diff options
author | Android Build Coastguard Worker <android-build-coastguard-worker@google.com> | 2023-03-02 16:11:06 +0000 |
---|---|---|
committer | Gerrit Code Review <noreply-gerritcodereview@google.com> | 2023-03-02 16:11:06 +0000 |
commit | 26a85afddc6972ca3f14f1ef66799bab7df9779a (patch) | |
tree | 226a0806ffbf27ec0b3b17abe93cb77c40424fca | |
parent | 651369e02869f1cd1962fcfd1f18b7b22920f40b (diff) | |
parent | a68429cd168db074240f1a1e81fbcfa1eea3cf4b (diff) | |
download | extras-26a85afddc6972ca3f14f1ef66799bab7df9779a.tar.gz |
Merge "Snap for 9679998 from c88c5f166e1c0ad615f6750efb46fdb84463f7ed to sdk-release" into sdk-releaseplatform-tools-34.0.1
43 files changed, 1011 insertions, 1308 deletions
diff --git a/partition_tools/lpmake.cc b/partition_tools/lpmake.cc index 76372daf..16dfec52 100644 --- a/partition_tools/lpmake.cc +++ b/partition_tools/lpmake.cc @@ -17,9 +17,11 @@ #include <getopt.h> #include <inttypes.h> #include <stdio.h> +#include <stdlib.h> #ifndef WIN32 #include <sysexits.h> #endif +#include <unistd.h> #include <algorithm> #include <memory> @@ -27,6 +29,7 @@ #include <android-base/parseint.h> #include <android-base/result.h> #include <android-base/strings.h> +#include <android-base/unique_fd.h> #include <liblp/builder.h> #include <liblp/liblp.h> @@ -35,12 +38,15 @@ using namespace android::fs_mgr; using android::base::Error; using android::base::Result; +using android::base::unique_fd; #ifdef WIN32 static constexpr int EX_OK = 0; static constexpr int EX_USAGE = 1; static constexpr int EX_SOFTWARE = 2; static constexpr int EX_CANTCREAT = 3; +#else +static constexpr int O_BINARY = 0; #endif /* Prints program usage to |where|. */ @@ -165,16 +171,47 @@ struct PartitionInfo { }; static uint64_t CalculateBlockDeviceSize(uint32_t alignment, uint32_t metadata_size, + uint32_t metadata_slots, const std::vector<PartitionInfo>& partitions) { - uint64_t ret = std::max(alignment, LP_PARTITION_RESERVED_BYTES + - (LP_METADATA_GEOMETRY_SIZE + metadata_size) * 2) + - partitions.size() * alignment; + uint64_t ret = LP_PARTITION_RESERVED_BYTES; + ret += LP_METADATA_GEOMETRY_SIZE * 2; + + // Each metadata slot has a primary and backup copy. + ret += metadata_slots * metadata_size * 2; + + if (alignment) { + uint64_t remainder = ret % alignment; + uint64_t to_add = alignment - remainder; + if (to_add > std::numeric_limits<uint64_t>::max() - ret) { + return 0; + } + ret += to_add; + } + + ret += partitions.size() * alignment; for (const auto& partition_info : partitions) { ret += partition_info.size; } return ret; } +static bool GetFileSize(const std::string& path, uint64_t* size) { + unique_fd fd(open(path.c_str(), O_RDONLY | O_BINARY)); + if (fd < 0) { + fprintf(stderr, "Could not open file: %s: %s\n", path.c_str(), strerror(errno)); + return false; + } + + auto offs = lseek(fd.get(), 0, SEEK_END); + if (offs < 0) { + fprintf(stderr, "Failed to seek file: %s: %s\n", path.c_str(), strerror(errno)); + return false; + } + + *size = offs; + return true; +} + int main(int argc, char* argv[]) { struct option options[] = { { "device-size", required_argument, nullptr, (int)Option::kDeviceSize }, @@ -347,7 +384,12 @@ int main(int argc, char* argv[]) { } if (auto_blockdevice_size) { - blockdevice_size = CalculateBlockDeviceSize(alignment, metadata_size, partitions); + blockdevice_size = + CalculateBlockDeviceSize(alignment, metadata_size, metadata_slots, partitions); + if (!blockdevice_size) { + fprintf(stderr, "Invalid block device parameters.\n"); + return EX_USAGE; + } } // Must specify a block device via the old method (--device-size etc) or @@ -425,13 +467,21 @@ int main(int argc, char* argv[]) { } } - for (const auto& partition_info : partitions) { + for (auto& partition_info : partitions) { Partition* partition = builder->AddPartition(partition_info.name, partition_info.group_name, partition_info.attribute_flags); if (!partition) { fprintf(stderr, "Could not add partition: %s\n", partition_info.name.c_str()); return EX_SOFTWARE; } + if (!partition_info.size) { + // Deduce the size automatically. + if (auto iter = images.find(partition_info.name); iter != images.end()) { + if (!GetFileSize(iter->second, &partition_info.size)) { + return EX_SOFTWARE; + } + } + } if (!builder->ResizePartition(partition, partition_info.size)) { fprintf(stderr, "Not enough space on device for partition %s with size %" PRIu64 "\n", partition_info.name.c_str(), partition_info.size); diff --git a/power_profile/camera_avg/LICENSE b/power_profile/camera_avg/LICENSE index 4f229463..35cf914e 100644 --- a/power_profile/camera_avg/LICENSE +++ b/power_profile/camera_avg/LICENSE @@ -1,6 +1,3 @@ -Apache License --------------- - Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -201,447 +198,3 @@ Apache License WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - -All image and audio files (including *.png, *.jpg, *.svg, *.mp3, *.wav -and *.ogg) are licensed under the CC-BY-NC license. All other files are -licensed under the Apache 2 license. - -CC-BY-NC License ----------------- - -Attribution-NonCommercial-ShareAlike 4.0 International - -======================================================================= - -Creative Commons Corporation ("Creative Commons") is not a law firm and -does not provide legal services or legal advice. Distribution of -Creative Commons public licenses does not create a lawyer-client or -other relationship. Creative Commons makes its licenses and related -information available on an "as-is" basis. Creative Commons gives no -warranties regarding its licenses, any material licensed under their -terms and conditions, or any related information. Creative Commons -disclaims all liability for damages resulting from their use to the -fullest extent possible. - -Using Creative Commons Public Licenses - -Creative Commons public licenses provide a standard set of terms and -conditions that creators and other rights holders may use to share -original works of authorship and other material subject to copyright -and certain other rights specified in the public license below. The -following considerations are for informational purposes only, are not -exhaustive, and do not form part of our licenses. - - Considerations for licensors: Our public licenses are - intended for use by those authorized to give the public - permission to use material in ways otherwise restricted by - copyright and certain other rights. Our licenses are - irrevocable. Licensors should read and understand the terms - and conditions of the license they choose before applying it. - Licensors should also secure all rights necessary before - applying our licenses so that the public can reuse the - material as expected. Licensors should clearly mark any - material not subject to the license. This includes other CC- - licensed material, or material used under an exception or - limitation to copyright. More considerations for licensors: - wiki.creativecommons.org/Considerations_for_licensors - - Considerations for the public: By using one of our public - licenses, a licensor grants the public permission to use the - licensed material under specified terms and conditions. If - the licensor's permission is not necessary for any reason--for - example, because of any applicable exception or limitation to - copyright--then that use is not regulated by the license. Our - licenses grant only permissions under copyright and certain - other rights that a licensor has authority to grant. Use of - the licensed material may still be restricted for other - reasons, including because others have copyright or other - rights in the material. A licensor may make special requests, - such as asking that all changes be marked or described. - Although not required by our licenses, you are encouraged to - respect those requests where reasonable. More_considerations - for the public: - wiki.creativecommons.org/Considerations_for_licensees - -======================================================================= - -Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International -Public License - -By exercising the Licensed Rights (defined below), You accept and agree -to be bound by the terms and conditions of this Creative Commons -Attribution-NonCommercial-ShareAlike 4.0 International Public License -("Public License"). To the extent this Public License may be -interpreted as a contract, You are granted the Licensed Rights in -consideration of Your acceptance of these terms and conditions, and the -Licensor grants You such rights in consideration of benefits the -Licensor receives from making the Licensed Material available under -these terms and conditions. - - -Section 1 -- Definitions. - - a. Adapted Material means material subject to Copyright and Similar - Rights that is derived from or based upon the Licensed Material - and in which the Licensed Material is translated, altered, - arranged, transformed, or otherwise modified in a manner requiring - permission under the Copyright and Similar Rights held by the - Licensor. For purposes of this Public License, where the Licensed - Material is a musical work, performance, or sound recording, - Adapted Material is always produced where the Licensed Material is - synched in timed relation with a moving image. - - b. Adapter's License means the license You apply to Your Copyright - and Similar Rights in Your contributions to Adapted Material in - accordance with the terms and conditions of this Public License. - - c. BY-NC-SA Compatible License means a license listed at - creativecommons.org/compatiblelicenses, approved by Creative - Commons as essentially the equivalent of this Public License. - - d. Copyright and Similar Rights means copyright and/or similar rights - closely related to copyright including, without limitation, - performance, broadcast, sound recording, and Sui Generis Database - Rights, without regard to how the rights are labeled or - categorized. For purposes of this Public License, the rights - specified in Section 2(b)(1)-(2) are not Copyright and Similar - Rights. - - e. Effective Technological Measures means those measures that, in the - absence of proper authority, may not be circumvented under laws - fulfilling obligations under Article 11 of the WIPO Copyright - Treaty adopted on December 20, 1996, and/or similar international - agreements. - - f. Exceptions and Limitations means fair use, fair dealing, and/or - any other exception or limitation to Copyright and Similar Rights - that applies to Your use of the Licensed Material. - - g. License Elements means the license attributes listed in the name - of a Creative Commons Public License. The License Elements of this - Public License are Attribution, NonCommercial, and ShareAlike. - - h. Licensed Material means the artistic or literary work, database, - or other material to which the Licensor applied this Public - License. - - i. Licensed Rights means the rights granted to You subject to the - terms and conditions of this Public License, which are limited to - all Copyright and Similar Rights that apply to Your use of the - Licensed Material and that the Licensor has authority to license. - - j. Licensor means the individual(s) or entity(ies) granting rights - under this Public License. - - k. NonCommercial means not primarily intended for or directed towards - commercial advantage or monetary compensation. For purposes of - this Public License, the exchange of the Licensed Material for - other material subject to Copyright and Similar Rights by digital - file-sharing or similar means is NonCommercial provided there is - no payment of monetary compensation in connection with the - exchange. - - l. Share means to provide material to the public by any means or - process that requires permission under the Licensed Rights, such - as reproduction, public display, public performance, distribution, - dissemination, communication, or importation, and to make material - available to the public including in ways that members of the - public may access the material from a place and at a time - individually chosen by them. - - m. Sui Generis Database Rights means rights other than copyright - resulting from Directive 96/9/EC of the European Parliament and of - the Council of 11 March 1996 on the legal protection of databases, - as amended and/or succeeded, as well as other essentially - equivalent rights anywhere in the world. - - n. You means the individual or entity exercising the Licensed Rights - under this Public License. Your has a corresponding meaning. - - -Section 2 -- Scope. - - a. License grant. - - 1. Subject to the terms and conditions of this Public License, - the Licensor hereby grants You a worldwide, royalty-free, - non-sublicensable, non-exclusive, irrevocable license to - exercise the Licensed Rights in the Licensed Material to: - - a. reproduce and Share the Licensed Material, in whole or - in part, for NonCommercial purposes only; and - - b. produce, reproduce, and Share Adapted Material for - NonCommercial purposes only. - - 2. Exceptions and Limitations. For the avoidance of doubt, where - Exceptions and Limitations apply to Your use, this Public - License does not apply, and You do not need to comply with - its terms and conditions. - - 3. Term. The term of this Public License is specified in Section - 6(a). - - 4. Media and formats; technical modifications allowed. The - Licensor authorizes You to exercise the Licensed Rights in - all media and formats whether now known or hereafter created, - and to make technical modifications necessary to do so. The - Licensor waives and/or agrees not to assert any right or - authority to forbid You from making technical modifications - necessary to exercise the Licensed Rights, including - technical modifications necessary to circumvent Effective - Technological Measures. For purposes of this Public License, - simply making modifications authorized by this Section 2(a) - (4) never produces Adapted Material. - - 5. Downstream recipients. - - a. Offer from the Licensor -- Licensed Material. Every - recipient of the Licensed Material automatically - receives an offer from the Licensor to exercise the - Licensed Rights under the terms and conditions of this - Public License. - - b. Additional offer from the Licensor -- Adapted Material. - Every recipient of Adapted Material from You - automatically receives an offer from the Licensor to - exercise the Licensed Rights in the Adapted Material - under the conditions of the Adapter's License You apply. - - c. No downstream restrictions. You may not offer or impose - any additional or different terms or conditions on, or - apply any Effective Technological Measures to, the - Licensed Material if doing so restricts exercise of the - Licensed Rights by any recipient of the Licensed - Material. - - 6. No endorsement. Nothing in this Public License constitutes or - may be construed as permission to assert or imply that You - are, or that Your use of the Licensed Material is, connected - with, or sponsored, endorsed, or granted official status by, - the Licensor or others designated to receive attribution as - provided in Section 3(a)(1)(A)(i). - - b. Other rights. - - 1. Moral rights, such as the right of integrity, are not - licensed under this Public License, nor are publicity, - privacy, and/or other similar personality rights; however, to - the extent possible, the Licensor waives and/or agrees not to - assert any such rights held by the Licensor to the limited - extent necessary to allow You to exercise the Licensed - Rights, but not otherwise. - - 2. Patent and trademark rights are not licensed under this - Public License. - - 3. To the extent possible, the Licensor waives any right to - collect royalties from You for the exercise of the Licensed - Rights, whether directly or through a collecting society - under any voluntary or waivable statutory or compulsory - licensing scheme. In all other cases the Licensor expressly - reserves any right to collect such royalties, including when - the Licensed Material is used other than for NonCommercial - purposes. - - -Section 3 -- License Conditions. - -Your exercise of the Licensed Rights is expressly made subject to the -following conditions. - - a. Attribution. - - 1. If You Share the Licensed Material (including in modified - form), You must: - - a. retain the following if it is supplied by the Licensor - with the Licensed Material: - - i. identification of the creator(s) of the Licensed - Material and any others designated to receive - attribution, in any reasonable manner requested by - the Licensor (including by pseudonym if - designated); - - ii. a copyright notice; - - iii. a notice that refers to this Public License; - - iv. a notice that refers to the disclaimer of - warranties; - - v. a URI or hyperlink to the Licensed Material to the - extent reasonably practicable; - - b. indicate if You modified the Licensed Material and - retain an indication of any previous modifications; and - - c. indicate the Licensed Material is licensed under this - Public License, and include the text of, or the URI or - hyperlink to, this Public License. - - 2. You may satisfy the conditions in Section 3(a)(1) in any - reasonable manner based on the medium, means, and context in - which You Share the Licensed Material. For example, it may be - reasonable to satisfy the conditions by providing a URI or - hyperlink to a resource that includes the required - information. - 3. If requested by the Licensor, You must remove any of the - information required by Section 3(a)(1)(A) to the extent - reasonably practicable. - - b. ShareAlike. - - In addition to the conditions in Section 3(a), if You Share - Adapted Material You produce, the following conditions also apply. - - 1. The Adapter's License You apply must be a Creative Commons - license with the same License Elements, this version or - later, or a BY-NC-SA Compatible License. - - 2. You must include the text of, or the URI or hyperlink to, the - Adapter's License You apply. You may satisfy this condition - in any reasonable manner based on the medium, means, and - context in which You Share Adapted Material. - - 3. You may not offer or impose any additional or different terms - or conditions on, or apply any Effective Technological - Measures to, Adapted Material that restrict exercise of the - rights granted under the Adapter's License You apply. - - -Section 4 -- Sui Generis Database Rights. - -Where the Licensed Rights include Sui Generis Database Rights that -apply to Your use of the Licensed Material: - - a. for the avoidance of doubt, Section 2(a)(1) grants You the right - to extract, reuse, reproduce, and Share all or a substantial - portion of the contents of the database for NonCommercial purposes - only; - - b. if You include all or a substantial portion of the database - contents in a database in which You have Sui Generis Database - Rights, then the database in which You have Sui Generis Database - Rights (but not its individual contents) is Adapted Material, - including for purposes of Section 3(b); and - - c. You must comply with the conditions in Section 3(a) if You Share - all or a substantial portion of the contents of the database. - -For the avoidance of doubt, this Section 4 supplements and does not -replace Your obligations under this Public License where the Licensed -Rights include other Copyright and Similar Rights. - - -Section 5 -- Disclaimer of Warranties and Limitation of Liability. - - a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE - EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS - AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF - ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS, - IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION, - WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR - PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS, - ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT - KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT - ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU. - - b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE - TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION, - NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT, - INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES, - COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR - USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN - ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR - DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR - IN PART, THIS LIMITATION MAY NOT APPLY TO YOU. - - c. The disclaimer of warranties and limitation of liability provided - above shall be interpreted in a manner that, to the extent - possible, most closely approximates an absolute disclaimer and - waiver of all liability. - - -Section 6 -- Term and Termination. - - a. This Public License applies for the term of the Copyright and - Similar Rights licensed here. However, if You fail to comply with - this Public License, then Your rights under this Public License - terminate automatically. - - b. Where Your right to use the Licensed Material has terminated under - Section 6(a), it reinstates: - - 1. automatically as of the date the violation is cured, provided - it is cured within 30 days of Your discovery of the - violation; or - - 2. upon express reinstatement by the Licensor. - - For the avoidance of doubt, this Section 6(b) does not affect any - right the Licensor may have to seek remedies for Your violations - of this Public License. - - c. For the avoidance of doubt, the Licensor may also offer the - Licensed Material under separate terms or conditions or stop - distributing the Licensed Material at any time; however, doing so - will not terminate this Public License. - - d. Sections 1, 5, 6, 7, and 8 survive termination of this Public - License. - - -Section 7 -- Other Terms and Conditions. - - a. The Licensor shall not be bound by any additional or different - terms or conditions communicated by You unless expressly agreed. - - b. Any arrangements, understandings, or agreements regarding the - Licensed Material not stated herein are separate from and - independent of the terms and conditions of this Public License. - - -Section 8 -- Interpretation. - - a. For the avoidance of doubt, this Public License does not, and - shall not be interpreted to, reduce, limit, restrict, or impose - conditions on any use of the Licensed Material that could lawfully - be made without permission under this Public License. - - b. To the extent possible, if any provision of this Public License is - deemed unenforceable, it shall be automatically reformed to the - minimum extent necessary to make it enforceable. If the provision - cannot be reformed, it shall be severed from this Public License - without affecting the enforceability of the remaining terms and - conditions. - - c. No term or condition of this Public License will be waived and no - failure to comply consented to unless expressly agreed to by the - Licensor. - - d. Nothing in this Public License constitutes or may be interpreted - as a limitation upon, or waiver of, any privileges and immunities - that apply to the Licensor or You, including from the legal - processes of any jurisdiction or authority. - -======================================================================= - -Creative Commons is not a party to its public licenses. -Notwithstanding, Creative Commons may elect to apply one of its public -licenses to material it publishes and in those instances will be -considered the "Licensor." Except for the limited purpose of indicating -that material is shared under a Creative Commons public license or as -otherwise permitted by the Creative Commons policies published at -creativecommons.org/policies, Creative Commons does not authorize the -use of the trademark "Creative Commons" or any other trademark or logo -of Creative Commons without its prior written consent including, -without limitation, in connection with any unauthorized modifications -to any of its public licenses or any other arrangements, -understandings, or agreements concerning use of licensed material. For -the avoidance of doubt, this paragraph does not form part of the public -licenses. - -Creative Commons may be contacted at creativecommons.org. - diff --git a/puncture_fs/Android.bp b/puncture_fs/Android.bp deleted file mode 100644 index aef7ba43..00000000 --- a/puncture_fs/Android.bp +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright (C) 2014 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package { - default_applicable_licenses: ["system_extras_puncture_fs_license"], -} - -// Added automatically by a large-scale-change -// See: http://go/android-license-faq -license { - name: "system_extras_puncture_fs_license", - visibility: [":__subpackages__"], - license_kinds: [ - "SPDX-license-identifier-Apache-2.0", - ], - license_text: [ - "NOTICE", - ], -} - -cc_binary { - name: "puncture_fs", - - srcs: ["puncture_fs.cpp"], - cflags: [ - "-Wall", - "-Werror", - ], - - shared_libs: [ - "libc", - "liblog", - "liblogwrap", - ], -} diff --git a/puncture_fs/NOTICE b/puncture_fs/NOTICE deleted file mode 100644 index 316b4eb5..00000000 --- a/puncture_fs/NOTICE +++ /dev/null @@ -1,190 +0,0 @@ - - Copyright (c) 2014, The Android Open Source Project - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - diff --git a/puncture_fs/puncture_fs.cpp b/puncture_fs/puncture_fs.cpp deleted file mode 100644 index 9c01e3b0..00000000 --- a/puncture_fs/puncture_fs.cpp +++ /dev/null @@ -1,266 +0,0 @@ -/* - * Copyright (C) 2014 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <assert.h> -#include <errno.h> -#include <fcntl.h> -#include <getopt.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <unistd.h> - -#include <logwrap/logwrap.h> -#include <sys/stat.h> -#include <sys/statvfs.h> -#include <utils/Log.h> - -#define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0])) - -#define MAX_IO_WRITE_CHUNK_SIZE 0x100000 - -#ifndef min -#define min(a,b) ((a) < (b) ? (a) : (b)) -#endif - -typedef unsigned long u64; - -static void usage(const char * const progname) { - fprintf(stderr, - "Usage: %s [-s <seed>] -h <hole size in bytes> -t <total hole size in bytes> " - "path\n", - progname); -} - -static u64 get_free_space(const char * const path) { - struct statvfs s; - - if (statvfs(path, &s) < 0) { - fprintf(stderr, "\nerrno: %d. Failed to get free disk space on %s\n", - errno, path); - return 0; - } else { - return (u64)s.f_bsize * (u64)s.f_bfree; - } -} - -static u64 get_random_num(const u64 start, const u64 end) { - if (end - start <= 0) - return start; - assert(RAND_MAX >= 0x7FFFFFFF); - if ((end - start) > 0x7FFFFFFF) - return start + (((u64)random() << 31) | (u64)random()) % (end - start); - return start + (random() % (end - start)); -} - -static char get_random_char() { - return 'A' + random() % ('Z' - 'A'); -} - -static bool create_unique_file(const char * const dir_path, const u64 size, - const u64 id, char * const base, - const u64 base_length) { - u64 length = 0; - int fd; - char file_path[FILENAME_MAX]; - bool ret = true; - - base[random() % min(base_length, size)] = get_random_char(); - - sprintf(file_path, "%s/file_%lu", dir_path, id); - fd = open(file_path, O_WRONLY | O_CREAT | O_SYNC, 0777); - if (fd < 0) { - // We suppress ENOSPC erros as that is common as we approach the - // last few MBs of the fs as we don't account for the size of the newly - // added meta data after the initial free space computation. - if (errno != 28) { - fprintf(stderr, "\nerrno: %d. Failed to create %s\n", errno, file_path); - } - return false; - } - while (length + base_length < size) { - if (write(fd, base, base_length) < 0) { - if (errno != 28) { - fprintf(stderr, "\nerrno: %d. Failed to write %lu bytes to %s\n", - errno, base_length, file_path); - } - ret = false; - goto done; - } - length += base_length; - } - if (write(fd, base, size - length) < 0) { - if (errno != 28) { - fprintf(stderr, "\nerrno: %d. Failed to write last %lu bytes to %s\n", - errno, size - length, file_path); - } - ret = false; - goto done; - } -done: - if (close(fd) < 0) { - fprintf(stderr, "\nFailed to close %s\n", file_path); - ret = false; - } - return ret; -} - -static bool create_unique_dir(char *dir, const char * const root_path) { - char random_string[15]; - int i; - - for (i = 0; i < 14; ++i) { - random_string[i] = get_random_char(); - } - random_string[14] = '\0'; - - sprintf(dir, "%s/%s", root_path, random_string); - - if (mkdir(dir, 0777) < 0) { - fprintf(stderr, "\nerrno: %d. Failed to create %s\n", errno, dir); - return false; - } - return true; -} - -static bool puncture_fs (const char * const path, const u64 total_size, - const u64 hole_size, const u64 total_hole_size) { - u64 increments = (hole_size * total_size) / total_hole_size; - u64 hole_max; - u64 starting_max = 0; - u64 ending_max = increments; - char stay_dir[FILENAME_MAX], delete_dir[FILENAME_MAX]; - const char* rm_bin_argv[] = { "/system/bin/rm", "-rf", ""}; - u64 file_id = 1; - char *base_file_data; - u64 i = 0; - - if (!create_unique_dir(stay_dir, path) || - !create_unique_dir(delete_dir, path)) { - return false; - } - - base_file_data = (char*) malloc(MAX_IO_WRITE_CHUNK_SIZE); - for (i = 0; i < MAX_IO_WRITE_CHUNK_SIZE; ++i) { - base_file_data[i] = get_random_char(); - } - fprintf(stderr, "\n"); - while (ending_max <= total_size) { - fprintf(stderr, "\rSTAGE 1/2: %d%% Complete", - (int) (100.0 * starting_max / total_size)); - hole_max = get_random_num(starting_max, ending_max); - - do { - hole_max = get_random_num(starting_max, ending_max); - } while (hole_max == starting_max); - - create_unique_file(stay_dir, - hole_max - starting_max, - file_id++, - base_file_data, - MAX_IO_WRITE_CHUNK_SIZE); - create_unique_file(delete_dir, - hole_size, - file_id++, - base_file_data, - MAX_IO_WRITE_CHUNK_SIZE); - - starting_max = hole_max + hole_size; - ending_max += increments; - } - create_unique_file(stay_dir, - (ending_max - increments - starting_max), - file_id++, - base_file_data, - MAX_IO_WRITE_CHUNK_SIZE); - fprintf(stderr, "\rSTAGE 1/2: 100%% Complete\n"); - fprintf(stderr, "\rSTAGE 2/2: 0%% Complete"); - free(base_file_data); - rm_bin_argv[2] = delete_dir; - if (logwrap_fork_execvp(ARRAY_SIZE(rm_bin_argv), rm_bin_argv, nullptr, - false, LOG_KLOG, false, nullptr) < 0) { - fprintf(stderr, "\nFailed to delete %s\n", rm_bin_argv[2]); - return false; - } - fprintf(stderr, "\rSTAGE 2/2: 100%% Complete\n"); - return true; -} - -int main (const int argc, char ** const argv) { - int opt; - int mandatory_opt; - char *path = NULL; - int seed = time(NULL); - - u64 total_size = 0; - u64 hole_size = 0; - u64 total_hole_size = 0; - - mandatory_opt = 2; - while ((opt = getopt(argc, argv, "s:h:t:")) != -1) { - switch(opt) { - case 's': - seed = atoi(optarg); - break; - case 'h': - hole_size = atoll(optarg); - mandatory_opt--; - break; - case 't': - total_hole_size = atoll(optarg); - mandatory_opt--; - break; - default: - usage(argv[0]); - exit(EXIT_FAILURE); - } - } - if (mandatory_opt) { - usage(argv[0]); - exit(EXIT_FAILURE); - } - if (optind >= argc) { - fprintf(stderr, "\nExpected path name after options.\n"); - usage(argv[0]); - exit(EXIT_FAILURE); - } - path = argv[optind++]; - - if (optind < argc) { - fprintf(stderr, "\nUnexpected argument: %s\n", argv[optind]); - usage(argv[0]); - exit(EXIT_FAILURE); - } - - srandom(seed); - fprintf(stderr, "\nRandom seed is: %d\n", seed); - - total_size = get_free_space(path); - if (!total_size) { - exit(EXIT_FAILURE); - } - if (total_size < total_hole_size || total_hole_size < hole_size) { - fprintf(stderr, "\nInvalid sizes: total available size should be " - "larger than total hole size which is larger than " - "hole size\n"); - exit(EXIT_FAILURE); - } - - if (!puncture_fs(path, total_size, hole_size, total_hole_size)) { - exit(EXIT_FAILURE); - } - return 0; -} diff --git a/simpleperf/Android.bp b/simpleperf/Android.bp index 7731dbd2..e44e12e1 100644 --- a/simpleperf/Android.bp +++ b/simpleperf/Android.bp @@ -141,6 +141,7 @@ cc_defaults { "libprocinfo", "libevent", "libc++fs", + "librustc_demangle_static", ], }, linux_glibc_x86_64: { @@ -186,6 +187,7 @@ cc_defaults { static_libs: [ "libc++fs", "libdexfile_support", + "librustc_demangle_static", ], runtime_libs: [ "libdexfile", // libdexfile_support dependency diff --git a/simpleperf/README.md b/simpleperf/README.md index c1d23688..50fd2432 100644 --- a/simpleperf/README.md +++ b/simpleperf/README.md @@ -1,6 +1,14 @@ # Simpleperf -This file is documentation for simpleperf maintainers. +Android Studio includes a graphical front end to Simpleperf, documented in +[Inspect CPU activity with CPU Profiler](https://developer.android.com/studio/profile/cpu-profiler). +Most users will prefer to use that instead of using Simpleperf directly. + +If you prefer to use the command line, Simpleperf is a versatile command-line +CPU profiling tool included in the NDK for Mac, Linux, and Windows. + +This file contains documentation for simpleperf maintainers. + There is also [user documentation](doc/README.md). ## Building new prebuilts diff --git a/simpleperf/RecordFilter.cpp b/simpleperf/RecordFilter.cpp index a9853237..34054286 100644 --- a/simpleperf/RecordFilter.cpp +++ b/simpleperf/RecordFilter.cpp @@ -248,8 +248,8 @@ RecordFilter::~RecordFilter() {} bool RecordFilter::ParseOptions(OptionValueMap& options) { for (bool exclude : {true, false}) { std::string prefix = exclude ? "--exclude-" : "--include-"; - for (const OptionValue& value : options.PullValues(prefix + "pid")) { - if (auto pids = GetTidsFromString(*value.str_value, false); pids) { + if (auto strs = options.PullStringValues(prefix + "pid"); !strs.empty()) { + if (auto pids = GetPidsFromStrings(strs, false, false); pids) { AddPids(pids.value(), exclude); } else { return false; diff --git a/simpleperf/RecordReadThread.cpp b/simpleperf/RecordReadThread.cpp index ebc942a5..ae632d38 100644 --- a/simpleperf/RecordReadThread.cpp +++ b/simpleperf/RecordReadThread.cpp @@ -236,6 +236,9 @@ RecordReadThread::RecordReadThread(size_t record_buffer_size, const perf_event_a } record_buffer_low_level_ = std::min(record_buffer_size / 4, kDefaultLowBufferLevel); record_buffer_critical_level_ = std::min(record_buffer_size / 6, kDefaultCriticalBufferLevel); + LOG(VERBOSE) << "user buffer size = " << record_buffer_size + << ", low_level size = " << record_buffer_low_level_ + << ", critical_level size = " << record_buffer_critical_level_; if (!allow_cutting_samples) { record_buffer_low_level_ = record_buffer_critical_level_; } diff --git a/simpleperf/cmd_debug_unwind.cpp b/simpleperf/cmd_debug_unwind.cpp index c78c3cf8..7af7aa38 100644 --- a/simpleperf/cmd_debug_unwind.cpp +++ b/simpleperf/cmd_debug_unwind.cpp @@ -447,13 +447,17 @@ class TestFileGenerator : public RecordFileProcessor { } } else if (feat_type == PerfFileFormat::FEAT_FILE || feat_type == PerfFileFormat::FEAT_FILE2) { - size_t read_pos = 0; + uint64_t read_pos = 0; FileFeature file_feature; - while (reader_->ReadFileFeature(read_pos, &file_feature)) { + bool error = false; + while (reader_->ReadFileFeature(read_pos, file_feature, error)) { if (kept_binaries_.count(file_feature.path) && !writer_->WriteFileFeature(file_feature)) { return false; } } + if (error) { + return false; + } } else if (feat_type == PerfFileFormat::FEAT_BUILD_ID) { std::vector<BuildIdRecord> build_ids = reader_->ReadBuildIdFeature(); std::vector<BuildIdRecord> write_build_ids; diff --git a/simpleperf/cmd_dumprecord.cpp b/simpleperf/cmd_dumprecord.cpp index 27ae7e3c..89e56b67 100644 --- a/simpleperf/cmd_dumprecord.cpp +++ b/simpleperf/cmd_dumprecord.cpp @@ -15,6 +15,7 @@ */ #include <inttypes.h> +#include <stdint.h> #include <map> #include <string> @@ -415,17 +416,21 @@ SymbolInfo DumpRecordCommand::GetSymbolInfo(uint32_t pid, uint32_t tid, uint64_t } bool DumpRecordCommand::DumpAuxData(const AuxRecord& aux) { + if (aux.data->aux_size > SIZE_MAX) { + LOG(ERROR) << "invalid aux size"; + return false; + } size_t size = aux.data->aux_size; if (size > 0) { - std::unique_ptr<uint8_t[]> data(new uint8_t[size]); - if (!record_file_reader_->ReadAuxData(aux.Cpu(), aux.data->aux_offset, data.get(), size)) { + std::vector<uint8_t> data; + if (!record_file_reader_->ReadAuxData(aux.Cpu(), aux.data->aux_offset, size, &data)) { return false; } if (!etm_decoder_) { LOG(ERROR) << "ETMDecoder isn't created"; return false; } - return etm_decoder_->ProcessData(data.get(), size, !aux.Unformatted(), aux.Cpu()); + return etm_decoder_->ProcessData(data.data(), size, !aux.Unformatted(), aux.Cpu()); } return true; } @@ -479,9 +484,10 @@ bool DumpRecordCommand::DumpFeatureSection() { PrintIndented(1, "cmdline: %s\n", android::base::Join(cmdline, ' ').c_str()); } else if (feature == FEAT_FILE || feature == FEAT_FILE2) { FileFeature file; - size_t read_pos = 0; + uint64_t read_pos = 0; + bool error = false; PrintIndented(1, "file:\n"); - while (record_file_reader_->ReadFileFeature(read_pos, &file)) { + while (record_file_reader_->ReadFileFeature(read_pos, file, error)) { PrintIndented(2, "file_path %s\n", file.path.c_str()); PrintIndented(2, "file_type %s\n", DsoTypeToString(file.type)); PrintIndented(2, "min_vaddr 0x%" PRIx64 "\n", file.min_vaddr); @@ -498,6 +504,9 @@ bool DumpRecordCommand::DumpFeatureSection() { } } } + if (error) { + return false; + } } else if (feature == FEAT_META_INFO) { PrintIndented(1, "meta_info:\n"); for (auto& pair : record_file_reader_->GetMetaInfoFeature()) { diff --git a/simpleperf/cmd_inject.cpp b/simpleperf/cmd_inject.cpp index f34cc33c..43cfc728 100644 --- a/simpleperf/cmd_inject.cpp +++ b/simpleperf/cmd_inject.cpp @@ -14,6 +14,7 @@ * limitations under the License. */ +#include <stdint.h> #include <stdio.h> #include <unistd.h> @@ -310,13 +311,14 @@ class PerfDataReader { } } else if (r->type() == PERF_RECORD_AUX) { AuxRecord* aux = static_cast<AuxRecord*>(r); - uint64_t aux_size = aux->data->aux_size; + if (aux->data->aux_size > SIZE_MAX) { + LOG(ERROR) << "invalid aux size"; + return false; + } + size_t aux_size = aux->data->aux_size; if (aux_size > 0) { - if (aux_data_buffer_.size() < aux_size) { - aux_data_buffer_.resize(aux_size); - } - if (!record_file_reader_->ReadAuxData(aux->Cpu(), aux->data->aux_offset, - aux_data_buffer_.data(), aux_size)) { + if (!record_file_reader_->ReadAuxData(aux->Cpu(), aux->data->aux_offset, aux_size, + &aux_data_buffer_)) { LOG(ERROR) << "failed to read aux data in " << filename_; return false; } diff --git a/simpleperf/cmd_merge.cpp b/simpleperf/cmd_merge.cpp index e80dd953..6f9bc20c 100644 --- a/simpleperf/cmd_merge.cpp +++ b/simpleperf/cmd_merge.cpp @@ -381,8 +381,9 @@ class MergeCommand : public Command { // Read file features. for (auto& reader : readers_) { FileFeature file; - size_t read_pos = 0; - while (reader->ReadFileFeature(read_pos, &file)) { + uint64_t read_pos = 0; + bool error = false; + while (reader->ReadFileFeature(read_pos, file, error)) { if (files_to_drop.count(file.path) != 0) { continue; } @@ -395,6 +396,9 @@ class MergeCommand : public Command { files_to_drop.emplace(file.path); } } + if (error) { + return false; + } } // Write file features. for (const auto& [file_path, file] : file_map) { diff --git a/simpleperf/cmd_record.cpp b/simpleperf/cmd_record.cpp index e69355d4..514f9061 100644 --- a/simpleperf/cmd_record.cpp +++ b/simpleperf/cmd_record.cpp @@ -111,8 +111,8 @@ constexpr size_t DEFAULT_CALL_CHAIN_JOINER_CACHE_SIZE = 8 * 1024 * 1024; // Currently, the record buffer size in user-space is set to match the kernel buffer size on a // 8 core system. For system-wide recording, it is 8K pages * 4K page_size * 8 cores = 256MB. // For non system-wide recording, it is 1K pages * 4K page_size * 8 cores = 64MB. -static constexpr size_t kRecordBufferSize = 64 * 1024 * 1024; -static constexpr size_t kSystemWideRecordBufferSize = 256 * 1024 * 1024; +static constexpr size_t kDefaultRecordBufferSize = 64 * 1024 * 1024; +static constexpr size_t kDefaultSystemWideRecordBufferSize = 256 * 1024 * 1024; static constexpr size_t kDefaultAuxBufferSize = 4 * 1024 * 1024; @@ -146,8 +146,9 @@ class RecordCommand : public Command { " On non-rooted devices, the app must be debuggable,\n" " because we use run-as to switch to the app's context.\n" #endif -"-p pid1,pid2,... Record events on existing processes. Mutually exclusive\n" -" with -a.\n" +"-p pid_or_process_name_regex1,pid_or_process_name_regex2,...\n" +" Record events on existing processes. Processes are searched either by pid\n" +" or process name regex. Mutually exclusive with -a.\n" "-t tid1,tid2,... Record events on existing threads. Mutually exclusive with -a.\n" "\n" "Select monitored event types:\n" @@ -215,9 +216,12 @@ class RecordCommand : public Command { " This option requires at least one branch type among any, any_call,\n" " any_ret, ind_call.\n" "-b Enable taken branch stack sampling. Same as '-j any'.\n" -"-m mmap_pages Set the size of the buffer used to receiving sample data from\n" -" the kernel. It should be a power of 2. If not set, the max\n" -" possible value <= 1024 will be used.\n" +"-m mmap_pages Set pages used in the kernel to cache sample data for each cpu.\n" +" It should be a power of 2. If not set, the max possible value <= 1024\n" +" will be used.\n" +"--user-buffer-size <buffer_size> Set buffer size in userspace to cache sample data.\n" +" By default, it is 64M for process recording and 256M\n" +" for system wide recording.\n" "--aux-buffer-size <buffer_size> Set aux buffer size, only used in cs-etm event type.\n" " Need to be power of 2 and page size aligned.\n" " Used memory size is (buffer_size * (cpu_count + 1).\n" @@ -405,6 +409,7 @@ RECORD_FILTER_OPTION_HELP_MSG_FOR_RECORDING EventSelectionSet event_selection_set_; std::pair<size_t, size_t> mmap_page_range_; + std::optional<size_t> user_buffer_size_; size_t aux_buffer_size_ = kDefaultAuxBufferSize; ThreadTree thread_tree_; @@ -607,8 +612,13 @@ bool RecordCommand::PrepareRecording(Workload* workload) { if (!event_selection_set_.OpenEventFiles(cpus_)) { return false; } - size_t record_buffer_size = - system_wide_collection_ ? kSystemWideRecordBufferSize : kRecordBufferSize; + size_t record_buffer_size = 0; + if (user_buffer_size_.has_value()) { + record_buffer_size = user_buffer_size_.value(); + } else { + record_buffer_size = + system_wide_collection_ ? kDefaultSystemWideRecordBufferSize : kDefaultRecordBufferSize; + } if (!event_selection_set_.MmapEventFiles(mmap_page_range_.first, mmap_page_range_.second, aux_buffer_size_, record_buffer_size, allow_cutting_samples_, exclude_perf_)) { @@ -992,8 +1002,8 @@ bool RecordCommand::ParseOptions(const std::vector<std::string>& args, out_fd_.reset(static_cast<int>(value->uint_value)); } - for (const OptionValue& value : options.PullValues("-p")) { - if (auto pids = GetTidsFromString(*value.str_value, true); pids) { + if (auto strs = options.PullStringValues("-p"); !strs.empty()) { + if (auto pids = GetPidsFromStrings(strs, true, true); pids) { event_selection_set_.AddMonitoredProcesses(pids.value()); } else { return false; @@ -1011,6 +1021,15 @@ bool RecordCommand::ParseOptions(const std::vector<std::string>& args, post_unwind_ = false; } + if (auto value = options.PullValue("--user-buffer-size"); value) { + uint64_t v = value->uint_value; + if (v > std::numeric_limits<size_t>::max() || v == 0) { + LOG(ERROR) << "invalid user buffer size: " << v; + return false; + } + user_buffer_size_ = static_cast<size_t>(v); + } + if (!options.PullUintValue("--size-limit", &size_limit_in_bytes_, 1)) { return false; } diff --git a/simpleperf/cmd_record_impl.h b/simpleperf/cmd_record_impl.h index 2ef407bf..846bfbc9 100644 --- a/simpleperf/cmd_record_impl.h +++ b/simpleperf/cmd_record_impl.h @@ -76,6 +76,7 @@ inline const OptionFormatMap& GetRecordCmdOptionFormats() { {"--post-unwind", {OptionValueType::NONE, OptionType::SINGLE, AppRunnerType::ALLOWED}}, {"--post-unwind=no", {OptionValueType::NONE, OptionType::SINGLE, AppRunnerType::ALLOWED}}, {"--post-unwind=yes", {OptionValueType::NONE, OptionType::SINGLE, AppRunnerType::ALLOWED}}, + {"--user-buffer-size", {OptionValueType::UINT, OptionType::SINGLE, AppRunnerType::ALLOWED}}, {"--size-limit", {OptionValueType::UINT, OptionType::SINGLE, AppRunnerType::ALLOWED}}, {"--start_profiling_fd", {OptionValueType::UINT, OptionType::SINGLE, AppRunnerType::CHECK_FD}}, diff --git a/simpleperf/cmd_record_test.cpp b/simpleperf/cmd_record_test.cpp index 9098fed6..a2409fac 100644 --- a/simpleperf/cmd_record_test.cpp +++ b/simpleperf/cmd_record_test.cpp @@ -352,14 +352,16 @@ static void ProcessSymbolsInPerfDataFile( auto reader = RecordFileReader::CreateInstance(perf_data_file); ASSERT_TRUE(reader); FileFeature file; - size_t read_pos = 0; - while (reader->ReadFileFeature(read_pos, &file)) { + uint64_t read_pos = 0; + bool error = false; + while (reader->ReadFileFeature(read_pos, file, error)) { for (const auto& symbol : file.symbols) { if (callback(symbol, file.type)) { return; } } } + ASSERT_FALSE(error); } // Check if dumped symbols in perf.data matches our expectation. @@ -712,8 +714,9 @@ class RecordingAppHelper { bool RecordData(const std::string& record_cmd) { std::vector<std::string> args = android::base::Split(record_cmd, " "); - args.emplace_back("-o"); - args.emplace_back(perf_data_file_.path); + // record_cmd may end with child command. We should put output options before it. + args.emplace(args.begin(), "-o"); + args.emplace(args.begin() + 1, perf_data_file_.path); return RecordCmd()->Run(args); } @@ -729,6 +732,8 @@ class RecordingAppHelper { return success; } + void DumpData() { CreateCommandInstance("report")->Run({"-i", perf_data_file_.path}); } + std::string GetDataPath() const { return perf_data_file_.path; } private: @@ -751,7 +756,10 @@ static void TestRecordingApps(const std::string& app_name, const std::string& ap return strstr(name, expected_class_name.c_str()) != nullptr && strstr(name, expected_method_name.c_str()) != nullptr; }; - ASSERT_TRUE(helper.CheckData(process_symbol)); + if (!helper.CheckData(process_symbol)) { + helper.DumpData(); + FAIL() << "Expected Java symbol doesn't exist in the profiling data"; + } // Check app_package_name and app_type. auto reader = RecordFileReader::CreateInstance(helper.GetDataPath()); @@ -766,7 +774,9 @@ static void TestRecordingApps(const std::string& app_name, const std::string& ap reader.reset(nullptr); // Check that simpleperf can't execute child command in app uid. - ASSERT_FALSE(helper.RecordData("--app " + app_name + " -e " + GetDefaultEvent() + " sleep 1")); + if (!IsRoot()) { + ASSERT_FALSE(helper.RecordData("--app " + app_name + " -e " + GetDefaultEvent() + " sleep 1")); + } } TEST(record_cmd, app_option_for_debuggable_app) { @@ -1238,3 +1248,7 @@ TEST(record_cmd, add_counter_option) { })); ASSERT_TRUE(has_sample); } + +TEST(record_cmd, user_buffer_size_option) { + ASSERT_TRUE(RunRecordCmd({"--user-buffer-size", "256M"})); +} diff --git a/simpleperf/cmd_report.cpp b/simpleperf/cmd_report.cpp index f68eddce..2f2dab10 100644 --- a/simpleperf/cmd_report.cpp +++ b/simpleperf/cmd_report.cpp @@ -662,8 +662,8 @@ bool ReportCommand::ParseOptions(const std::vector<std::string>& args) { return false; } - for (const OptionValue& value : options.PullValues("--pids")) { - if (auto pids = GetTidsFromString(*value.str_value, false); pids) { + if (auto strs = options.PullStringValues("--pids"); !strs.empty()) { + if (auto pids = GetPidsFromStrings(strs, false, false); pids) { record_filter_.AddPids(pids.value(), false); } else { return false; diff --git a/simpleperf/cmd_report_test.cpp b/simpleperf/cmd_report_test.cpp index bdaf8975..edca413f 100644 --- a/simpleperf/cmd_report_test.cpp +++ b/simpleperf/cmd_report_test.cpp @@ -227,7 +227,7 @@ TEST_F(ReportCommandTest, wrong_pid_filter_option) { Report(PERF_DATA_WITH_MULTIPLE_PIDS_AND_TIDS, {"--pids", "2,bogus"}); exit(success ? 0 : 1); }, - testing::ExitedWithCode(1), "Invalid tid 'bogus'"); + testing::ExitedWithCode(1), "invalid pid: bogus"); } TEST_F(ReportCommandTest, tid_filter_option) { diff --git a/simpleperf/cmd_stat.cpp b/simpleperf/cmd_stat.cpp index 8a5b1264..009afc3c 100644 --- a/simpleperf/cmd_stat.cpp +++ b/simpleperf/cmd_stat.cpp @@ -380,8 +380,10 @@ class StatCommand : public Command { "-o output_filename Write report to output_filename instead of standard output.\n" "--per-core Print counters for each cpu core.\n" "--per-thread Print counters for each thread.\n" -"-p pid1,pid2,... Stat events on existing processes. Mutually exclusive with -a.\n" -"-t tid1,tid2,... Stat events on existing threads. Mutually exclusive with -a.\n" +"-p pid_or_process_name_regex1,pid_or_process_name_regex2,...\n" +" Stat events on existing processes. Processes are searched either by pid\n" +" or process name regex. Mutually exclusive with -a.\n" +"-t tid1,tid2,... Stat events on existing threads. Mutually exclusive with -a.\n" "--print-hw-counter Test and print CPU PMU hardware counters available on the device.\n" "--sort key1,key2,... Select keys used to sort the report, used when --per-thread\n" " or --per-core appears. The appearance order of keys decides\n" @@ -690,8 +692,8 @@ bool StatCommand::ParseOptions(const std::vector<std::string>& args, report_per_core_ = options.PullBoolValue("--per-core"); report_per_thread_ = options.PullBoolValue("--per-thread"); - for (const OptionValue& value : options.PullValues("-p")) { - if (auto pids = GetTidsFromString(*value.str_value, true); pids) { + if (auto strs = options.PullStringValues("-p"); !strs.empty()) { + if (auto pids = GetPidsFromStrings(strs, true, true); pids) { event_selection_set_.AddMonitoredProcesses(pids.value()); } else { return false; diff --git a/simpleperf/command.h b/simpleperf/command.h index a46da6f5..1616bc40 100644 --- a/simpleperf/command.h +++ b/simpleperf/command.h @@ -117,6 +117,14 @@ struct OptionValueMap { return res; } + std::vector<std::string> PullStringValues(const OptionName& name) { + std::vector<std::string> res; + for (const auto& value : PullValues(name)) { + res.emplace_back(*value.str_value); + } + return res; + } + std::vector<OptionValue> PullValues(const OptionName& name) { auto pair = values.equal_range(name); if (pair.first != pair.second) { diff --git a/simpleperf/doc/README.md b/simpleperf/doc/README.md index 51ffc839..f989b785 100644 --- a/simpleperf/doc/README.md +++ b/simpleperf/doc/README.md @@ -1,5 +1,9 @@ # Simpleperf +Android Studio includes a graphical front end to Simpleperf, documented in +[Inspect CPU activity with CPU Profiler](https://developer.android.com/studio/profile/cpu-profiler). +Most users will prefer to use that instead of using Simpleperf directly. + Simpleperf is a native CPU profiling tool for Android. It can be used to profile both Android applications and native processes running on Android. It can profile both Java and C++ code on Android. The simpleperf executable can run on Android >=L, @@ -57,6 +61,7 @@ Python scripts are split into three parts according to their functions: 3. Scripts used for parsing profiling data, like simpleperf_report_lib.py. +The python scripts are tested on Python >= 3.9. Older versions may not be supported. Detailed documentation for the Python scripts is [here](#scripts-reference). @@ -238,3 +243,13 @@ $ mmma system/extras/simpleperf -j30 If built successfully, out/target/product/generic_arm64/system/bin/simpleperf is for ARM64, and out/target/product/generic_arm64/system/bin/simpleperf32 is for ARM. + +The source code of simpleperf python scripts is in [system/extras/simpleperf/scripts](https://android.googlesource.com/platform/system/extras/+/master/simpleperf/scripts/). +Most scripts rely on simpleperf binaries to work. To update binaries for scripts (using linux +x86_64 host and android arm64 target as an example): +```sh +$ cp out/host/linux-x86/lib64/libsimpleperf_report.so system/extras/simpleperf/scripts/bin/linux/x86_64/libsimpleperf_report.so +$ cp out/target/product/generic_arm64/system/bin/simpleperf_ndk64 system/extras/simpleperf/scripts/bin/android/arm64/simpleperf +``` + +Then you can try the latest simpleperf scripts and binaries in system/extras/simpleperf/scripts. diff --git a/simpleperf/doc/executable_commands_reference.md b/simpleperf/doc/executable_commands_reference.md index e3db970a..0e29fbf7 100644 --- a/simpleperf/doc/executable_commands_reference.md +++ b/simpleperf/doc/executable_commands_reference.md @@ -206,15 +206,23 @@ process to run the new command and then monitor the child process. # Stat process 11904 and 11905. $ simpleperf stat -p 11904,11905 --duration 10 +# Stat processes with name containing "chrome". +$ simpleperf stat -p chrome --duration 10 +# Stat processes with name containing part matching regex "chrome:(privileged|sandboxed)". +$ simpleperf stat -p "chrome:(privileged|sandboxed)" --duration 10 + # Stat thread 11904 and 11905. $ simpleperf stat -t 11904,11905 --duration 10 # Start a child process running `ls`, and stat it. $ simpleperf stat ls -# Stat the process of an Android application. This only works for debuggable apps on non-rooted -# devices. -$ simpleperf stat --app simpleperf.example.cpp +# Stat the process of an Android application. On non-root devices, this only works for debuggable +# or profileable from shell apps. +$ simpleperf stat --app simpleperf.example.cpp --duration 10 + +# Stat only selected thread 11904 in an app. +$ simpleperf stat --app simpleperf.example.cpp -t 11904 --duration 10 # Stat system wide using -a. $ simpleperf stat -a --duration 10 @@ -362,15 +370,23 @@ The way to select target in record command is similar to that in the stat comman # Record process 11904 and 11905. $ simpleperf record -p 11904,11905 --duration 10 +# Record processes with name containing "chrome". +$ simpleperf record -p chrome --duration 10 +# Record processes with name containing part matching regex "chrome:(privileged|sandboxed)". +$ simpleperf record -p "chrome:(privileged|sandboxed)" --duration 10 + # Record thread 11904 and 11905. $ simpleperf record -t 11904,11905 --duration 10 # Record a child process running `ls`. $ simpleperf record ls -# Record the process of an Android application. This only works for debuggable apps on non-rooted -# devices. -$ simpleperf record --app simpleperf.example.cpp +# Record the process of an Android application. On non-root devices, this only works for debuggable +# or profileable from shell apps. +$ simpleperf record --app simpleperf.example.cpp --duration 10 + +# Record only selected thread 11904 in an app. +$ simpleperf record --app simpleperf.example.cpp -t 11904 --duration 10 # Record system wide. $ simpleperf record -a --duration 10 diff --git a/simpleperf/dso.cpp b/simpleperf/dso.cpp index 386febab..32b6731e 100644 --- a/simpleperf/dso.cpp +++ b/simpleperf/dso.cpp @@ -267,6 +267,9 @@ void Dso::SetDemangle(bool demangle) { } extern "C" char* __cxa_demangle(const char* mangled_name, char* buf, size_t* n, int* status); +#if defined(__linux__) || defined(__darwin__) +extern "C" char* rustc_demangle(const char* mangled, char* out, size_t* len, int* status); +#endif std::string Dso::Demangle(const std::string& name) { if (!demangle_) { @@ -278,19 +281,35 @@ std::string Dso::Demangle(const std::string& name) { if (is_linker_symbol) { mangled_str += linker_prefix.size(); } - std::string result = name; - char* demangled_name = __cxa_demangle(mangled_str, nullptr, nullptr, &status); - if (status == 0) { - if (is_linker_symbol) { - result = std::string("[linker]") + demangled_name; - } else { - result = demangled_name; + + if (mangled_str[0] == '_') { + char* demangled_name = nullptr; + int status = -2; // -2 means name didn't demangle. + if (mangled_str[1] == 'Z') { + demangled_name = __cxa_demangle(mangled_str, nullptr, nullptr, &status); +#if defined(__linux__) || defined(__darwin__) + } else if (mangled_str[1] == 'R') { + demangled_name = rustc_demangle(mangled_str, nullptr, nullptr, &status); +#endif + } + if (status == 0) { + // demangled successfully + std::string result; + if (is_linker_symbol) { + result = std::string("[linker]") + demangled_name; + } else { + result = demangled_name; + } + free(demangled_name); + return result; } - free(demangled_name); - } else if (is_linker_symbol) { - result = std::string("[linker]") + mangled_str; } - return result; + + // failed to demangle + if (is_linker_symbol) { + return std::string("[linker]") + mangled_str; + } + return name; } bool Dso::SetSymFsDir(const std::string& symfs_dir) { @@ -624,7 +643,8 @@ class ElfDso : public Dso { } if ((status == ElfStatus::FILE_NOT_FOUND || status == ElfStatus::FILE_MALFORMED) && build_id.IsEmpty()) { - // This is likely to be a file wongly thought of as an ELF file, maybe due to stack unwinding. + // This is likely to be a file wongly thought of as an ELF file, maybe due to stack + // unwinding. log_level = android::base::DEBUG; } ReportReadElfSymbolResult(status, path_, GetDebugFilePath(), log_level); @@ -870,10 +890,9 @@ class KernelModuleDso : public Dso { // need to know its relative position in the module memory. There are two ways: // 1. Read the kernel module file to calculate the relative position of .text section. It // is relatively complex and depends on both PLT entries and the kernel version. - // 2. Find a module symbol in .text section, get its address in memory from /proc/kallsyms, and - // its vaddr_in_file from the kernel module file. Then other symbols in .text section can be - // mapped in the same way. - // Below we use the second method. + // 2. Find a module symbol in .text section, get its address in memory from /proc/kallsyms, + // and its vaddr_in_file from the kernel module file. Then other symbols in .text section can + // be mapped in the same way. Below we use the second method. // 1. Select a module symbol in /proc/kallsyms. kernel_dso_->LoadSymbols(); diff --git a/simpleperf/dso_test.cpp b/simpleperf/dso_test.cpp index 5b62d051..11bcde19 100644 --- a/simpleperf/dso_test.cpp +++ b/simpleperf/dso_test.cpp @@ -369,3 +369,12 @@ TEST(dso, read_symbol_warning) { ASSERT_EQ(capture.str().find("failed to read symbols"), std::string::npos); } } + +TEST(dso, demangle) { + ASSERT_EQ(Dso::Demangle("main"), "main"); + ASSERT_EQ(Dso::Demangle("_ZN4main4main17h2a68d4d833d7495aE"), "main::main::h2a68d4d833d7495a"); +#if defined(__linux__) || defined(__darwin__) + // Demangling rust symbol is only supported on linux and darwin. + ASSERT_EQ(Dso::Demangle("_RNvC6_123foo3bar"), "123foo::bar"); +#endif +} diff --git a/simpleperf/environment.cpp b/simpleperf/environment.cpp index 436dc9b4..8c4b9576 100644 --- a/simpleperf/environment.cpp +++ b/simpleperf/environment.cpp @@ -487,16 +487,10 @@ std::set<pid_t> WaitForAppProcesses(const std::string& package_name) { while (true) { std::vector<pid_t> pids = GetAllProcesses(); for (pid_t pid : pids) { - std::string argv0; - if (!android::base::ReadFileToString("/proc/" + std::to_string(pid) + "/cmdline", &argv0)) { - // Maybe we don't have permission to read it. + std::string process_name = GetCompleteProcessName(pid); + if (process_name.empty()) { continue; } - size_t pos = argv0.find('\0'); - if (pos != std::string::npos) { - argv0.resize(pos); - } - std::string process_name = android::base::Basename(argv0); // The app may have multiple processes, with process name like // com.google.android.googlequicksearchbox:search. size_t split_pos = process_name.find(':'); @@ -829,9 +823,22 @@ void AllowMoreOpenedFiles() { // On Android <= O, the hard limit is 4096, and the soft limit is 1024. // On Android >= P, both the hard and soft limit are 32768. rlimit limit; - if (getrlimit(RLIMIT_NOFILE, &limit) == 0) { - limit.rlim_cur = limit.rlim_max; - setrlimit(RLIMIT_NOFILE, &limit); + if (getrlimit(RLIMIT_NOFILE, &limit) != 0) { + return; + } + rlim_t new_limit = limit.rlim_max; + if (IsRoot()) { + rlim_t sysctl_nr_open = 0; + if (ReadUintFromProcFile("/proc/sys/fs/nr_open", &sysctl_nr_open) && + sysctl_nr_open > new_limit) { + new_limit = sysctl_nr_open; + } + } + if (limit.rlim_cur < new_limit) { + limit.rlim_cur = limit.rlim_max = new_limit; + if (setrlimit(RLIMIT_NOFILE, &limit) == 0) { + LOG(DEBUG) << "increased open file limit to " << new_limit; + } } } @@ -939,18 +946,16 @@ bool MappedFileOnlyExistInMemory(const char* filename) { } std::string GetCompleteProcessName(pid_t pid) { - std::string s; - if (!android::base::ReadFileToString(android::base::StringPrintf("/proc/%d/cmdline", pid), &s)) { - s.clear(); + std::string argv0; + if (!android::base::ReadFileToString("/proc/" + std::to_string(pid) + "/cmdline", &argv0)) { + // Maybe we don't have permission to read it. + return std::string(); } - for (size_t i = 0; i < s.size(); ++i) { - // /proc/pid/cmdline uses 0 to separate arguments. - if (isspace(s[i]) || s[i] == 0) { - s.resize(i); - break; - } + size_t pos = argv0.find('\0'); + if (pos != std::string::npos) { + argv0.resize(pos); } - return s; + return android::base::Basename(argv0); } const char* GetTraceFsDir() { diff --git a/simpleperf/record_file.h b/simpleperf/record_file.h index 0f436a2e..6daae288 100644 --- a/simpleperf/record_file.h +++ b/simpleperf/record_file.h @@ -183,10 +183,11 @@ class RecordFileReader { // File feature section contains many file information. This function reads // one file information located at [read_pos]. [read_pos] is 0 at the first - // call, and is updated to point to the next file information. Return true - // if read successfully, and return false if there is no more file - // information. - bool ReadFileFeature(size_t& read_pos, FileFeature* file); + // call, and is updated to point to the next file information. + // When read successfully, return true and set error to false. + // When no more data to read, return false and set error to false. + // When having error, return false and set error to true. + bool ReadFileFeature(uint64_t& read_pos, FileFeature& file, bool& error); const std::unordered_map<std::string, std::string>& GetMetaInfoFeature() { return meta_info_; } std::string GetClockId(); @@ -194,7 +195,7 @@ class RecordFileReader { bool LoadBuildIdAndFileFeatures(ThreadTree& thread_tree); - bool ReadAuxData(uint32_t cpu, uint64_t aux_offset, void* buf, size_t size); + bool ReadAuxData(uint32_t cpu, uint64_t aux_offset, size_t size, std::vector<uint8_t>* buf); bool Close(); @@ -209,8 +210,8 @@ class RecordFileReader { bool ReadAttrSection(); bool ReadIdsForAttr(const PerfFileFormat::FileAttr& attr, std::vector<uint64_t>* ids); bool ReadFeatureSectionDescriptors(); - bool ReadFileV1Feature(size_t& read_pos, FileFeature* file); - bool ReadFileV2Feature(size_t& read_pos, FileFeature* file); + bool ReadFileV1Feature(uint64_t& read_pos, uint64_t max_size, FileFeature& file); + bool ReadFileV2Feature(uint64_t& read_pos, uint64_t max_size, FileFeature& file); bool ReadMetaInfoFeature(); void UseRecordingEnvironment(); std::unique_ptr<Record> ReadRecord(); diff --git a/simpleperf/record_file_reader.cpp b/simpleperf/record_file_reader.cpp index d6ad62c7..166917c8 100644 --- a/simpleperf/record_file_reader.cpp +++ b/simpleperf/record_file_reader.cpp @@ -541,107 +541,106 @@ std::vector<uint64_t> RecordFileReader::ReadAuxTraceFeature() { return reader.error ? std::vector<uint64_t>() : auxtrace_offset; } -bool RecordFileReader::ReadFileFeature(size_t& read_pos, FileFeature* file) { - file->Clear(); - bool result = false; - if (HasFeature(FEAT_FILE)) { - result = ReadFileV1Feature(read_pos, file); - } else if (HasFeature(FEAT_FILE2)) { - result = ReadFileV2Feature(read_pos, file); - } - if (!result) { - LOG(ERROR) << "failed to read file feature section"; - } - return result; -} - -bool RecordFileReader::ReadFileV1Feature(size_t& read_pos, FileFeature* file) { - auto it = feature_section_descriptors_.find(FEAT_FILE); - if (it == feature_section_descriptors_.end()) { +bool RecordFileReader::ReadFileFeature(uint64_t& read_pos, FileFeature& file, bool& error) { + file.Clear(); + error = false; + + bool use_v1 = false; + PerfFileFormat::SectionDesc desc; + if (auto it = feature_section_descriptors_.find(FEAT_FILE); + it != feature_section_descriptors_.end()) { + use_v1 = true; + desc = it->second; + } else if (auto it = feature_section_descriptors_.find(FEAT_FILE2); + it != feature_section_descriptors_.end()) { + desc = it->second; + } else { return false; } - if (read_pos >= it->second.size) { + + if (read_pos >= desc.size) { return false; } if (read_pos == 0) { - if (fseek(record_fp_, it->second.offset, SEEK_SET) != 0) { + if (fseek(record_fp_, desc.offset, SEEK_SET) != 0) { PLOG(ERROR) << "fseek() failed"; + error = true; return false; } } + + bool result = false; + if (use_v1) { + result = ReadFileV1Feature(read_pos, desc.size - read_pos, file); + } else { + result = ReadFileV2Feature(read_pos, desc.size - read_pos, file); + } + if (!result) { + LOG(ERROR) << "failed to read file feature section"; + error = true; + } + return result; +} + +bool RecordFileReader::ReadFileV1Feature(uint64_t& read_pos, uint64_t max_size, FileFeature& file) { uint32_t size = 0; - if (!Read(&size, 4) || size > it->second.size) { + if (max_size < 4 || !Read(&size, 4) || max_size - 4 < size) { return false; } + read_pos += 4; std::vector<char> buf(size); if (!Read(buf.data(), size)) { return false; } - read_pos += 4 + size; + read_pos += size; BinaryReader reader(buf.data(), buf.size()); - file->path = reader.ReadString(); + file.path = reader.ReadString(); uint32_t file_type = 0; reader.Read(file_type); if (file_type > DSO_UNKNOWN_FILE) { - LOG(ERROR) << "unknown file type for " << file->path + LOG(ERROR) << "unknown file type for " << file.path << " in file feature section: " << file_type; return false; } - file->type = static_cast<DsoType>(file_type); - reader.Read(file->min_vaddr); + file.type = static_cast<DsoType>(file_type); + reader.Read(file.min_vaddr); uint32_t symbol_count = 0; reader.Read(symbol_count); if (symbol_count > size) { return false; } - file->symbols.reserve(symbol_count); + file.symbols.reserve(symbol_count); while (symbol_count-- > 0) { uint64_t start_vaddr = 0; uint32_t len = 0; reader.Read(start_vaddr); reader.Read(len); std::string name = reader.ReadString(); - file->symbols.emplace_back(name, start_vaddr, len); + file.symbols.emplace_back(name, start_vaddr, len); } - if (file->type == DSO_DEX_FILE) { + if (file.type == DSO_DEX_FILE) { uint32_t offset_count = 0; reader.Read(offset_count); if (offset_count > size) { return false; } - file->dex_file_offsets.resize(offset_count); - reader.Read(file->dex_file_offsets.data(), offset_count); + file.dex_file_offsets.resize(offset_count); + reader.Read(file.dex_file_offsets.data(), offset_count); } - file->file_offset_of_min_vaddr = std::numeric_limits<uint64_t>::max(); - if ((file->type == DSO_ELF_FILE || file->type == DSO_KERNEL_MODULE) && !reader.error && + file.file_offset_of_min_vaddr = std::numeric_limits<uint64_t>::max(); + if ((file.type == DSO_ELF_FILE || file.type == DSO_KERNEL_MODULE) && !reader.error && reader.LeftSize() > 0) { - reader.Read(file->file_offset_of_min_vaddr); + reader.Read(file.file_offset_of_min_vaddr); } return !reader.error && reader.LeftSize() == 0; } -bool RecordFileReader::ReadFileV2Feature(size_t& read_pos, FileFeature* file) { - auto it = feature_section_descriptors_.find(FEAT_FILE2); - if (it == feature_section_descriptors_.end()) { - return false; - } - if (read_pos >= it->second.size) { - return false; - } - if (read_pos == 0) { - if (fseek(record_fp_, it->second.offset, SEEK_SET) != 0) { - PLOG(ERROR) << "fseek() failed"; - return false; - } - } +bool RecordFileReader::ReadFileV2Feature(uint64_t& read_pos, uint64_t max_size, FileFeature& file) { uint32_t size; - if (!Read(&size, 4)) { + if (max_size < 4 || !Read(&size, 4) || max_size - 4 < size) { return false; } read_pos += 4; - if (read_pos > it->second.size || size > it->second.size - read_pos) { - return false; - } std::string s(size, '\0'); if (!Read(s.data(), size)) { return false; @@ -651,31 +650,31 @@ bool RecordFileReader::ReadFileV2Feature(size_t& read_pos, FileFeature* file) { if (!proto_file.ParseFromString(s)) { return false; } - file->path = proto_file.path(); - file->type = static_cast<DsoType>(proto_file.type()); - file->min_vaddr = proto_file.min_vaddr(); - file->symbols.reserve(proto_file.symbol_size()); + file.path = proto_file.path(); + file.type = static_cast<DsoType>(proto_file.type()); + file.min_vaddr = proto_file.min_vaddr(); + file.symbols.reserve(proto_file.symbol_size()); for (size_t i = 0; i < proto_file.symbol_size(); i++) { const auto& proto_symbol = proto_file.symbol(i); - file->symbols.emplace_back(proto_symbol.name(), proto_symbol.vaddr(), proto_symbol.len()); + file.symbols.emplace_back(proto_symbol.name(), proto_symbol.vaddr(), proto_symbol.len()); } - if (file->type == DSO_DEX_FILE) { + if (file.type == DSO_DEX_FILE) { if (!proto_file.has_dex_file()) { return false; } const auto& dex_file_offsets = proto_file.dex_file().dex_file_offset(); - file->dex_file_offsets.insert(file->dex_file_offsets.end(), dex_file_offsets.begin(), - dex_file_offsets.end()); - } else if (file->type == DSO_ELF_FILE) { + file.dex_file_offsets.insert(file.dex_file_offsets.end(), dex_file_offsets.begin(), + dex_file_offsets.end()); + } else if (file.type == DSO_ELF_FILE) { if (!proto_file.has_elf_file()) { return false; } - file->file_offset_of_min_vaddr = proto_file.elf_file().file_offset_of_min_vaddr(); - } else if (file->type == DSO_KERNEL_MODULE) { + file.file_offset_of_min_vaddr = proto_file.elf_file().file_offset_of_min_vaddr(); + } else if (file.type == DSO_KERNEL_MODULE) { if (!proto_file.has_kernel_module()) { return false; } - file->file_offset_of_min_vaddr = proto_file.kernel_module().memory_offset_of_min_vaddr(); + file.file_offset_of_min_vaddr = proto_file.kernel_module().memory_offset_of_min_vaddr(); } return true; } @@ -742,24 +741,29 @@ bool RecordFileReader::LoadBuildIdAndFileFeatures(ThreadTree& thread_tree) { } Dso::SetBuildIds(build_ids); - if (HasFeature(PerfFileFormat::FEAT_FILE) || HasFeature(PerfFileFormat::FEAT_FILE2)) { - FileFeature file_feature; - size_t read_pos = 0; - while (ReadFileFeature(read_pos, &file_feature)) { - if (!thread_tree.AddDsoInfo(file_feature)) { - return false; - } + FileFeature file_feature; + uint64_t read_pos = 0; + bool error = false; + while (ReadFileFeature(read_pos, file_feature, error)) { + if (!thread_tree.AddDsoInfo(file_feature)) { + return false; } } - return true; + return !error; } -bool RecordFileReader::ReadAuxData(uint32_t cpu, uint64_t aux_offset, void* buf, size_t size) { +bool RecordFileReader::ReadAuxData(uint32_t cpu, uint64_t aux_offset, size_t size, + std::vector<uint8_t>* buf) { long saved_pos = ftell(record_fp_); if (saved_pos == -1) { PLOG(ERROR) << "ftell() failed"; return false; } + OverflowResult aux_end = SafeAdd(aux_offset, size); + if (aux_end.overflow) { + LOG(ERROR) << "aux_end overflow"; + return false; + } if (aux_data_location_.empty() && !BuildAuxDataLocation()) { return false; } @@ -772,7 +776,7 @@ bool RecordFileReader::ReadAuxData(uint32_t cpu, uint64_t aux_offset, void* buf, auto location_it = std::upper_bound(it->second.begin(), it->second.end(), aux_offset, comp); if (location_it != it->second.begin()) { --location_it; - if (location_it->aux_offset + location_it->aux_size >= aux_offset + size) { + if (location_it->aux_offset + location_it->aux_size >= aux_end.value) { location = &*location_it; } } @@ -782,7 +786,10 @@ bool RecordFileReader::ReadAuxData(uint32_t cpu, uint64_t aux_offset, void* buf, << aux_offset << ", size " << size; return false; } - if (!ReadAtOffset(aux_offset - location->aux_offset + location->file_offset, buf, size)) { + if (buf->size() < size) { + buf->resize(size); + } + if (!ReadAtOffset(aux_offset - location->aux_offset + location->file_offset, buf->data(), size)) { return false; } if (fseek(record_fp_, saved_pos, SEEK_SET) != 0) { @@ -807,8 +814,27 @@ bool RecordFileReader::BuildAuxDataLocation() { if (!auxtrace.Parse(file_attrs_[0].attr, buf.get(), buf.get() + AuxTraceRecord::Size())) { return false; } - aux_data_location_[auxtrace.data->cpu].emplace_back( - auxtrace.data->offset, auxtrace.data->aux_size, offset + auxtrace.size()); + AuxDataLocation location(auxtrace.data->offset, auxtrace.data->aux_size, + offset + auxtrace.size()); + OverflowResult aux_end = SafeAdd(location.aux_offset, location.aux_size); + OverflowResult file_end = SafeAdd(location.file_offset, location.aux_size); + if (aux_end.overflow || file_end.overflow || file_end.value > file_size_) { + LOG(ERROR) << "invalid auxtrace feature section"; + return false; + } + auto location_it = aux_data_location_.find(auxtrace.data->cpu); + if (location_it != aux_data_location_.end()) { + const AuxDataLocation& prev_location = location_it->second.back(); + uint64_t prev_aux_end = prev_location.aux_offset + prev_location.aux_size; + // The AuxTraceRecords should be sorted by aux_offset for each cpu. + if (prev_aux_end > location.aux_offset) { + LOG(ERROR) << "invalid auxtrace feature section"; + return false; + } + location_it->second.emplace_back(location); + } else { + aux_data_location_[auxtrace.data->cpu].emplace_back(location); + } } return true; } diff --git a/simpleperf/record_file_test.cpp b/simpleperf/record_file_test.cpp index bf0436ad..395b6603 100644 --- a/simpleperf/record_file_test.cpp +++ b/simpleperf/record_file_test.cpp @@ -218,15 +218,16 @@ TEST_F(RecordFileTest, write_file2_feature_section) { // Read from a record file. std::unique_ptr<RecordFileReader> reader = RecordFileReader::CreateInstance(tmpfile_.path); ASSERT_TRUE(reader != nullptr); - size_t read_pos = 0; + uint64_t read_pos = 0; FileFeature file; + bool error = false; auto check_symbol = [](const Symbol& sym1, const Symbol& sym2) { return sym1.addr == sym2.addr && sym1.len == sym2.len && strcmp(sym1.Name(), sym2.Name()) == 0; }; size_t file_id = 0; - while (reader->ReadFileFeature(read_pos, &file)) { + while (reader->ReadFileFeature(read_pos, file, error)) { ASSERT_LT(file_id, files.size()); const FileFeature& expected_file = files[file_id++]; ASSERT_EQ(file.path, expected_file.path); @@ -253,5 +254,6 @@ TEST_F(RecordFileTest, write_file2_feature_section) { ASSERT_EQ(file.file_offset_of_min_vaddr, expected_file.file_offset_of_min_vaddr); } } + ASSERT_FALSE(error); ASSERT_EQ(file_id, files.size()); }
\ No newline at end of file diff --git a/simpleperf/record_file_writer.cpp b/simpleperf/record_file_writer.cpp index 83e47a4e..bcc26a33 100644 --- a/simpleperf/record_file_writer.cpp +++ b/simpleperf/record_file_writer.cpp @@ -368,7 +368,13 @@ bool RecordFileWriter::WriteFileFeature(const FileFeature& file) { proto::FileFeature::Symbol* proto_symbol = proto_file.add_symbol(); proto_symbol->set_vaddr(symbol.addr); proto_symbol->set_len(symbol.len); - proto_symbol->set_name(symbol.Name()); + // Store demangled names for rust symbols. Because simpleperf on windows host doesn't know + // how to demangle them. + if (strncmp(symbol.Name(), "_R", 2) == 0) { + proto_symbol->set_name(symbol.DemangledName()); + } else { + proto_symbol->set_name(symbol.Name()); + } }; for (const Symbol& symbol : file.symbols) { write_symbol(symbol); diff --git a/simpleperf/report_utils.cpp b/simpleperf/report_utils.cpp index 74959374..ecbc6aa8 100644 --- a/simpleperf/report_utils.cpp +++ b/simpleperf/report_utils.cpp @@ -16,6 +16,9 @@ #include "report_utils.h" +#include <stdlib.h> + +#include <android-base/parsebool.h> #include <android-base/scopeguard.h> #include <android-base/strings.h> @@ -186,6 +189,21 @@ static bool IsArtEntry(const CallChainReportEntry& entry, bool* is_jni_trampolin return false; }; +CallChainReportBuilder::CallChainReportBuilder(ThreadTree& thread_tree) + : thread_tree_(thread_tree) { + const char* env_name = "REMOVE_R8_SYNTHESIZED_FRAME"; + const char* s = getenv(env_name); + if (s != nullptr) { + auto result = android::base::ParseBool(s); + if (result == android::base::ParseBoolResult::kError) { + LOG(WARNING) << "invalid value in env variable " << env_name; + } else if (result == android::base::ParseBoolResult::kTrue) { + LOG(INFO) << "R8 synthesized frames will be removed."; + remove_r8_synthesized_frame_ = true; + } + } +} + bool CallChainReportBuilder::AddProguardMappingFile(std::string_view mapping_file) { if (!retrace_) { retrace_.reset(new ProguardMappingRetrace); @@ -349,7 +367,7 @@ void CallChainReportBuilder::DeObfuscateJavaMethods(std::vector<CallChainReportE std::string original_name; bool synthesized; if (retrace_->DeObfuscateJavaMethods(name, &original_name, &synthesized)) { - if (synthesized) { + if (synthesized && remove_r8_synthesized_frame_) { callchain.erase(callchain.begin() + i); continue; } diff --git a/simpleperf/report_utils.h b/simpleperf/report_utils.h index dbf95270..fa42bba6 100644 --- a/simpleperf/report_utils.h +++ b/simpleperf/report_utils.h @@ -93,7 +93,7 @@ struct CallChainReportEntry { class CallChainReportBuilder { public: - CallChainReportBuilder(ThreadTree& thread_tree) : thread_tree_(thread_tree) {} + CallChainReportBuilder(ThreadTree& thread_tree); // If true, remove interpreter frames both before and after a Java frame. // Default is true. void SetRemoveArtFrame(bool enable) { remove_art_frame_ = enable; } @@ -119,6 +119,7 @@ class CallChainReportBuilder { ThreadTree& thread_tree_; bool remove_art_frame_ = true; + bool remove_r8_synthesized_frame_ = false; bool convert_jit_frame_ = true; bool java_method_initialized_ = false; std::unordered_map<std::string, JavaMethod> java_method_map_; diff --git a/simpleperf/report_utils_test.cpp b/simpleperf/report_utils_test.cpp index 7b393fb5..55771218 100644 --- a/simpleperf/report_utils_test.cpp +++ b/simpleperf/report_utils_test.cpp @@ -16,6 +16,8 @@ #include <gtest/gtest.h> +#include <stdlib.h> + #include <string> #include <vector> @@ -362,13 +364,50 @@ TEST_F(CallChainReportBuilderTest, add_proguard_mapping_file) { ASSERT_EQ(entries[2].execution_type, CallChainExecutionType::JIT_JVM_METHOD); } -TEST_F(CallChainReportBuilderTest, add_proguard_mapping_file_synthesized_frame) { +TEST_F(CallChainReportBuilderTest, not_remove_synthesized_frame_by_default) { + std::vector<uint64_t> fake_ips = { + 0x2200, // 2200, // obfuscated_class.obfuscated_java_method + 0x3200, // 3200, // obfuscated_class.obfuscated_java_method2 + }; + + TemporaryFile tmpfile; + ASSERT_TRUE(android::base::WriteStringToFile( + "android.support.v4.app.RemoteActionCompatParcelizer -> obfuscated_class:\n" + " 13:13:androidx.core.app.RemoteActionCompat read(androidx.versionedparcelable.Versioned" + "Parcel) -> obfuscated_java_method\n" + " # {\"id\":\"com.android.tools.r8.synthesized\"}\n" + " 13:13:androidx.core.app.RemoteActionCompat " + "android.support.v4.app.RemoteActionCompatParcelizer.read2(androidx.versionedparcelable." + "VersionedParcel) -> obfuscated_java_method2", + tmpfile.path)); + + // By default, synthesized frames are kept. + CallChainReportBuilder builder(thread_tree); + builder.AddProguardMappingFile(tmpfile.path); + std::vector<CallChainReportEntry> entries = builder.Build(thread, fake_ips, 0); + ASSERT_EQ(entries.size(), 2); + ASSERT_EQ(entries[0].ip, 0x2200); + ASSERT_STREQ(entries[0].symbol->DemangledName(), + "android.support.v4.app.RemoteActionCompatParcelizer.read"); + ASSERT_EQ(entries[0].dso->Path(), fake_dex_file_path); + ASSERT_EQ(entries[0].vaddr_in_file, 0x200); + ASSERT_EQ(entries[0].execution_type, CallChainExecutionType::INTERPRETED_JVM_METHOD); + ASSERT_EQ(entries[1].ip, 0x3200); + ASSERT_STREQ(entries[1].symbol->DemangledName(), + "android.support.v4.app.RemoteActionCompatParcelizer.read2"); + ASSERT_EQ(entries[1].dso->Path(), fake_jit_cache_path); + ASSERT_EQ(entries[1].vaddr_in_file, 0x3200); + ASSERT_EQ(entries[1].execution_type, CallChainExecutionType::JIT_JVM_METHOD); +} + +TEST_F(CallChainReportBuilderTest, remove_synthesized_frame_with_env_variable) { + // Windows doesn't support setenv and unsetenv. So don't test on it. +#if !defined(__WIN32) std::vector<uint64_t> fake_ips = { 0x2200, // 2200, // obfuscated_class.obfuscated_java_method 0x3200, // 3200, // obfuscated_class.obfuscated_java_method2 }; - // Synthesized frames are removed. TemporaryFile tmpfile; ASSERT_TRUE(android::base::WriteStringToFile( "android.support.v4.app.RemoteActionCompatParcelizer -> obfuscated_class:\n" @@ -379,7 +418,11 @@ TEST_F(CallChainReportBuilderTest, add_proguard_mapping_file_synthesized_frame) "android.support.v4.app.RemoteActionCompatParcelizer.read2(androidx.versionedparcelable." "VersionedParcel) -> obfuscated_java_method2", tmpfile.path)); + + // With environment variable set, synthesized frames are removed. + ASSERT_EQ(setenv("REMOVE_R8_SYNTHESIZED_FRAME", "1", 1), 0); CallChainReportBuilder builder(thread_tree); + ASSERT_EQ(unsetenv("REMOVE_R8_SYNTHESIZED_FRAME"), 0); builder.AddProguardMappingFile(tmpfile.path); std::vector<CallChainReportEntry> entries = builder.Build(thread, fake_ips, 0); ASSERT_EQ(entries.size(), 1); @@ -389,6 +432,7 @@ TEST_F(CallChainReportBuilderTest, add_proguard_mapping_file_synthesized_frame) ASSERT_EQ(entries[0].dso->Path(), fake_jit_cache_path); ASSERT_EQ(entries[0].vaddr_in_file, 0x3200); ASSERT_EQ(entries[0].execution_type, CallChainExecutionType::JIT_JVM_METHOD); +#endif // !defined(__WIN32) } TEST_F(CallChainReportBuilderTest, add_proguard_mapping_file_for_jit_method_with_signature) { diff --git a/simpleperf/scripts/binary_cache_builder.py b/simpleperf/scripts/binary_cache_builder.py index 4322e2c0..1b09e7ad 100755 --- a/simpleperf/scripts/binary_cache_builder.py +++ b/simpleperf/scripts/binary_cache_builder.py @@ -19,13 +19,14 @@ it, and put them in binary_cache. """ -from dataclasses import dataclass +from collections import defaultdict import logging import os import os.path from pathlib import Path import shutil -from typing import List, Optional, Union +import sys +from typing import Dict, List, Optional, Tuple, Union from simpleperf_report_lib import ReportLib from simpleperf_utils import ( @@ -37,27 +38,232 @@ def is_jit_symfile(dso_name): return dso_name.split('/')[-1].startswith('TemporaryFile') -class BinaryCacheBuilder(object): +class BinaryCache: + def __init__(self, binary_dir: Path): + self.binary_dir = binary_dir + + def get_path_in_cache(self, device_path: str, build_id: str) -> Path: + """ Given a binary path in perf.data, return its corresponding path in the cache. + """ + if build_id: + filename = device_path.split('/')[-1] + # Add build id to make the filename unique. + unique_filename = build_id[2:] + '-' + filename + return self.binary_dir / unique_filename + + # For elf file without build id, we can only follow its path on device. Otherwise, + # simpleperf can't find it. However, we don't prefer this way. Because: + # 1) It doesn't work for native libs loaded directly from apk + # (android:extractNativeLibs=”false”). + # 2) It may exceed path limit on windows. + if device_path.startswith('/'): + device_path = device_path[1:] + device_path = device_path.replace('/', os.sep) + return Path(os.path.join(self.binary_dir, device_path)) + + +class BinarySource: + """ Source to find debug binaries. """ + + def __init__(self, readelf: ReadElf): + self.readelf = readelf + + def collect_binaries(self, binaries: Dict[str, str], binary_cache: BinaryCache): + """ pull binaries needed in perf.data to binary_cache. + binaries: maps from binary path to its build_id in perf.data. + """ + raise Exception('not implemented') + + def read_build_id(self, path: Path): + return self.readelf.get_build_id(path) + + +class BinarySourceFromDevice(BinarySource): + """ Pull binaries from device. """ + + def __init__(self, readelf: ReadElf, disable_adb_root: bool): + super().__init__(readelf) + self.adb = AdbHelper(enable_switch_to_root=not disable_adb_root) + + def collect_binaries(self, binaries: Dict[str, str], binary_cache: BinaryCache): + if not self.adb.is_device_available(): + return + for path, build_id in binaries.items(): + self.collect_binary(path, build_id, binary_cache) + self.pull_kernel_symbols(binary_cache.binary_dir / 'kallsyms') + + def collect_binary(self, path: str, build_id: str, binary_cache: BinaryCache): + if not path.startswith('/') or path == "//anon" or path.startswith("/dev/"): + # [kernel.kallsyms] or unknown, or something we can't find binary. + return + binary_cache_file = binary_cache.get_path_in_cache(path, build_id) + self.check_and_pull_binary(path, build_id, binary_cache_file) + + def check_and_pull_binary(self, path: str, expected_build_id: str, binary_cache_file: Path): + """If the binary_cache_file exists and has the expected_build_id, there + is no need to pull the binary from device. Otherwise, pull it. + """ + if binary_cache_file.is_file() and ( + not expected_build_id or expected_build_id == self.read_build_id(binary_cache_file) + ): + logging.info('use current file in binary_cache: %s', binary_cache_file) + else: + logging.info('pull file to binary_cache: %s to %s', path, binary_cache_file) + target_dir = binary_cache_file.parent + if not target_dir.is_dir(): + os.makedirs(target_dir) + if binary_cache_file.is_file(): + binary_cache_file.unlink() + self.pull_file_from_device(path, binary_cache_file) + + def pull_file_from_device(self, device_path: str, host_path: Path): + if self.adb.run(['pull', device_path, str(host_path)]): + return True + # On non-root devices, we can't pull /data/app/XXX/base.odex directly. + # Instead, we can first copy the file to /data/local/tmp, then pull it. + filename = device_path[device_path.rfind('/')+1:] + if (self.adb.run(['shell', 'cp', device_path, '/data/local/tmp']) and + self.adb.run(['pull', '/data/local/tmp/' + filename, host_path])): + self.adb.run(['shell', 'rm', '/data/local/tmp/' + filename]) + return True + logging.warning('failed to pull %s from device', device_path) + return False + + def pull_kernel_symbols(self, file_path: Path): + if file_path.is_file(): + file_path.unlink() + if self.adb.switch_to_root(): + self.adb.run(['shell', 'echo', '0', '>/proc/sys/kernel/kptr_restrict']) + self.adb.run(['pull', '/proc/kallsyms', file_path]) + + +class BinarySourceFromLibDirs(BinarySource): + """ Collect binaries from lib dirs. """ + + def __init__(self, readelf: ReadElf, lib_dirs: List[Path]): + super().__init__(readelf) + self.lib_dirs = lib_dirs + self.filename_map = None + self.build_id_map = None + self.binary_cache = None + + def collect_binaries(self, binaries: Dict[str, str], binary_cache: BinaryCache): + self.create_filename_map(binaries) + self.create_build_id_map(binaries) + self.binary_cache = binary_cache + + # Search all files in lib_dirs, and copy matching files to build_cache. + for lib_dir in self.lib_dirs: + if self.is_platform_symbols_dir(lib_dir): + self.search_platform_symbols_dir(lib_dir) + else: + self.search_dir(lib_dir) + + def create_filename_map(self, binaries: Dict[str, str]): + """ Create a map mapping from filename to binaries having the name. """ + self.filename_map = defaultdict(list) + for path, build_id in binaries.items(): + index = path.rfind('/') + filename = path[index + 1:] + self.filename_map[filename].append((path, build_id)) + + def create_build_id_map(self, binaries: Dict[str, str]): + """ Create a map mapping from build id to binary path. """ + self.build_id_map = {} + for path, build_id in binaries.items(): + if build_id: + self.build_id_map[build_id] = path + + def is_platform_symbols_dir(self, lib_dir: Path): + """ Check if lib_dir points to $ANDROID_PRODUCT_OUT/symbols. """ + subdir_names = [p.name for p in lib_dir.iterdir()] + return lib_dir.name == 'symbols' and 'system' in subdir_names + + def search_platform_symbols_dir(self, lib_dir: Path): + """ Platform symbols dir contains too many binaries. Reading build ids for + all of them takes a long time. So we only read build ids for binaries + having names exist in filename_map. + """ + for root, _, files in os.walk(lib_dir): + for filename in files: + binaries = self.filename_map.get(filename) + if not binaries: + continue + file_path = Path(os.path.join(root, filename)) + build_id = self.read_build_id(file_path) + for path, expected_build_id in binaries: + if expected_build_id == build_id: + self.copy_to_binary_cache(file_path, build_id, path) + + def search_dir(self, lib_dir: Path): + """ For a normal lib dir, it's unlikely to contain many binaries. So we can read + build ids for all binaries in it. But users may give debug binaries with a name + different from the one recorded in perf.data. So we should only rely on build id + if it is available. + """ + for root, _, files in os.walk(lib_dir): + for filename in files: + file_path = Path(os.path.join(root, filename)) + build_id = self.read_build_id(file_path) + if build_id: + # For elf file with build id, use build id to match. + device_path = self.build_id_map.get(build_id) + if device_path: + self.copy_to_binary_cache(file_path, build_id, device_path) + elif self.readelf.is_elf_file(file_path): + # For elf file without build id, use filename to match. + for path, expected_build_id in self.filename_map.get(filename, []): + if not expected_build_id: + self.copy_to_binary_cache(file_path, '', path) + break + + def copy_to_binary_cache( + self, from_path: Path, expected_build_id: str, device_path: str): + to_path = self.binary_cache.get_path_in_cache(device_path, expected_build_id) + if not self.need_to_copy(from_path, to_path, expected_build_id): + # The existing file in binary_cache can provide more information, so no need to copy. + return + to_dir = to_path.parent + if not to_dir.is_dir(): + os.makedirs(to_dir) + logging.info('copy to binary_cache: %s to %s', from_path, to_path) + shutil.copy(from_path, to_path) + + def need_to_copy(self, from_path: Path, to_path: Path, expected_build_id: str): + if not to_path.is_file() or self.read_build_id(to_path) != expected_build_id: + return True + return self.get_file_stripped_level(from_path) < self.get_file_stripped_level(to_path) + + def get_file_stripped_level(self, path: Path) -> int: + """Return stripped level of an ELF file. Larger value means more stripped.""" + sections = self.readelf.get_sections(path) + if '.debug_line' in sections: + return 0 + if '.symtab' in sections: + return 1 + return 2 + + +class BinaryCacheBuilder: """Collect all binaries needed by perf.data in binary_cache.""" def __init__(self, ndk_path: Optional[str], disable_adb_root: bool): - self.adb = AdbHelper(enable_switch_to_root=not disable_adb_root) self.readelf = ReadElf(ndk_path) - self.binary_cache_dir = 'binary_cache' - if not os.path.isdir(self.binary_cache_dir): - os.makedirs(self.binary_cache_dir) + self.device_source = BinarySourceFromDevice(self.readelf, disable_adb_root) + self.binary_cache_dir = Path('binary_cache') + self.binary_cache = BinaryCache(self.binary_cache_dir) self.binaries = {} - def build_binary_cache(self, perf_data_path: str, symfs_dirs: List[Union[Path, str]]): + def build_binary_cache(self, perf_data_path: str, symfs_dirs: List[Union[Path, str]]) -> bool: self.collect_used_binaries(perf_data_path) - self.copy_binaries_from_symfs_dirs(symfs_dirs) - if self.adb.is_device_available(): - self.pull_binaries_from_device() - self._pull_kernel_symbols() + if not self.copy_binaries_from_symfs_dirs(symfs_dirs): + return False + self.pull_binaries_from_device() self.create_build_id_list() + return True def collect_used_binaries(self, perf_data_path): - """read perf.data, collect all used binaries and their build id (if available).""" + """read perf.data, collect all used binaries and their build id(if available).""" # A dict mapping from binary name to build_id binaries = {} lib = ReportLib() @@ -82,149 +288,45 @@ class BinaryCacheBuilder(object): binaries[name] = lib.GetBuildIdForPath(dso_name) self.binaries = binaries - def copy_binaries_from_symfs_dirs(self, symfs_dirs: List[Union[Path, str]]): - """collect all files in symfs_dirs.""" - if not symfs_dirs: - return - - # It is possible that the path of the binary in symfs_dirs doesn't match - # the one recorded in perf.data. For example, a file in symfs_dirs might - # be "debug/arm/obj/armeabi-v7a/libsudo-game-jni.so", but the path in - # perf.data is "/data/app/xxxx/lib/arm/libsudo-game-jni.so". So we match - # binaries if they have the same filename (like libsudo-game-jni.so) - # and same build_id. - - # Map from filename to binary paths. - filename_dict = {} - for binary in self.binaries: - index = binary.rfind('/') - filename = binary[index+1:] - paths = filename_dict.get(filename) - if paths is None: - filename_dict[filename] = paths = [] - paths.append(binary) - - # Walk through all files in symfs_dirs, and copy matching files to build_cache. - for symfs_dir in symfs_dirs: - for root, _, files in os.walk(symfs_dir): - for filename in files: - paths = filename_dict.get(filename) - if not paths: - continue - build_id = self._read_build_id(os.path.join(root, filename)) - for binary in paths: - expected_build_id = self.binaries.get(binary) - if expected_build_id == build_id: - self._copy_to_binary_cache(os.path.join(root, filename), - expected_build_id, binary) - break - - def _copy_to_binary_cache(self, from_path, expected_build_id, target_file): - if target_file[0] == '/': - target_file = target_file[1:] - target_file = target_file.replace('/', os.sep) - target_file = os.path.join(self.binary_cache_dir, target_file) - if not self._need_to_copy(from_path, target_file, expected_build_id): - # The existing file in binary_cache can provide more information, so no need to copy. - return - target_dir = os.path.dirname(target_file) - if not os.path.isdir(target_dir): - os.makedirs(target_dir) - logging.info('copy to binary_cache: %s to %s' % (from_path, target_file)) - shutil.copy(from_path, target_file) - - def _need_to_copy(self, source_file, target_file, expected_build_id): - if not os.path.isfile(target_file): - return True - if self._read_build_id(target_file) != expected_build_id: - return True - return self._get_file_stripped_level(source_file) < self._get_file_stripped_level( - target_file) - - def _get_file_stripped_level(self, file_path): - """Return stripped level of an ELF file. Larger value means more stripped.""" - sections = self.readelf.get_sections(file_path) - if '.debug_line' in sections: - return 0 - if '.symtab' in sections: - return 1 - return 2 + def copy_binaries_from_symfs_dirs(self, symfs_dirs: List[Union[str, Path]]) -> bool: + if symfs_dirs: + lib_dirs: List[Path] = [] + for symfs_dir in symfs_dirs: + if isinstance(symfs_dir, str): + symfs_dir = Path(symfs_dir) + if not symfs_dir.is_dir(): + logging.error("can't find dir %s", symfs_dir) + return False + lib_dirs.append(symfs_dir) + lib_dir_source = BinarySourceFromLibDirs(self.readelf, lib_dirs) + lib_dir_source.collect_binaries(self.binaries, self.binary_cache) + return True def pull_binaries_from_device(self): - """pull binaries needed in perf.data to binary_cache.""" - for binary in self.binaries: - build_id = self.binaries[binary] - if not binary.startswith('/') or binary == "//anon" or binary.startswith("/dev/"): - # [kernel.kallsyms] or unknown, or something we can't find binary. - continue - binary_cache_file = binary[1:].replace('/', os.sep) - binary_cache_file = os.path.join(self.binary_cache_dir, binary_cache_file) - self._check_and_pull_binary(binary, build_id, binary_cache_file) - - def _check_and_pull_binary(self, binary, expected_build_id, binary_cache_file): - """If the binary_cache_file exists and has the expected_build_id, there - is no need to pull the binary from device. Otherwise, pull it. - """ - need_pull = True - if os.path.isfile(binary_cache_file): - need_pull = False - if expected_build_id: - build_id = self._read_build_id(binary_cache_file) - if expected_build_id != build_id: - need_pull = True - if need_pull: - target_dir = os.path.dirname(binary_cache_file) - if not os.path.isdir(target_dir): - os.makedirs(target_dir) - if os.path.isfile(binary_cache_file): - os.remove(binary_cache_file) - logging.info('pull file to binary_cache: %s to %s' % (binary, binary_cache_file)) - self._pull_file_from_device(binary, binary_cache_file) - else: - logging.info('use current file in binary_cache: %s' % binary_cache_file) - - def _read_build_id(self, file_path): - """read build id of a binary on host.""" - return self.readelf.get_build_id(file_path) - - def _pull_file_from_device(self, device_path, host_path): - if self.adb.run(['pull', device_path, host_path]): - return True - # In non-root device, we can't pull /data/app/XXX/base.odex directly. - # Instead, we can first copy the file to /data/local/tmp, then pull it. - filename = device_path[device_path.rfind('/')+1:] - if (self.adb.run(['shell', 'cp', device_path, '/data/local/tmp']) and - self.adb.run(['pull', '/data/local/tmp/' + filename, host_path])): - self.adb.run(['shell', 'rm', '/data/local/tmp/' + filename]) - return True - logging.warning('failed to pull %s from device' % device_path) - return False - - def _pull_kernel_symbols(self): - file_path = os.path.join(self.binary_cache_dir, 'kallsyms') - if os.path.isfile(file_path): - os.remove(file_path) - if self.adb.switch_to_root(): - self.adb.run(['shell', 'echo', '0', '>/proc/sys/kernel/kptr_restrict']) - self.adb.run(['pull', '/proc/kallsyms', file_path]) + self.device_source.collect_binaries(self.binaries, self.binary_cache) def create_build_id_list(self): """ Create build_id_list. So report scripts can find a binary by its build_id instead of path. """ - build_id_list_path = os.path.join(self.binary_cache_dir, 'build_id_list') + build_id_list_path = self.binary_cache_dir / 'build_id_list' + # Write in binary mode to avoid "\r\n" problem on windows, which can confuse simpleperf. with open(build_id_list_path, 'wb') as fh: for root, _, files in os.walk(self.binary_cache_dir): for filename in files: - path = os.path.join(root, filename) - relative_path = path[len(self.binary_cache_dir) + 1:] - build_id = self._read_build_id(path) + path = Path(os.path.join(root, filename)) + build_id = self.readelf.get_build_id(path) if build_id: + relative_path = path.relative_to(self.binary_cache_dir) line = f'{build_id}={relative_path}\n' fh.write(str_to_bytes(line)) + def find_path_in_cache(self, device_path: str) -> Optional[Path]: + build_id = self.binaries.get(device_path) + return self.binary_cache.get_path_in_cache(device_path, build_id) + -def main(): +def main() -> bool: parser = BaseArgumentParser(description=""" Pull binaries needed by perf.data from device to binary_cache directory.""") parser.add_argument('-i', '--perf_data_path', default='perf.data', type=extant_file, help=""" @@ -238,8 +340,8 @@ def main(): ndk_path = None if not args.ndk_path else args.ndk_path[0] builder = BinaryCacheBuilder(ndk_path, args.disable_adb_root) symfs_dirs = flatten_arg_list(args.native_lib_dir) - builder.build_binary_cache(args.perf_data_path, symfs_dirs) + return builder.build_binary_cache(args.perf_data_path, symfs_dirs) if __name__ == '__main__': - main() + sys.exit(0 if main() else 1) diff --git a/simpleperf/scripts/gecko_profile_generator.py b/simpleperf/scripts/gecko_profile_generator.py index 396c9a58..9fe9ad3d 100755 --- a/simpleperf/scripts/gecko_profile_generator.py +++ b/simpleperf/scripts/gecko_profile_generator.py @@ -25,13 +25,15 @@ Then open gecko-profile.json.gz in https://profiler.firefox.com/ """ +from collections import Counter +from dataclasses import dataclass, field import json +import logging import sys +from typing import List, Dict, Optional, NamedTuple, Tuple -from dataclasses import dataclass, field from simpleperf_report_lib import ReportLib -from simpleperf_utils import BaseArgumentParser, flatten_arg_list, ReportLibOptions -from typing import List, Dict, Optional, NamedTuple, Set, Tuple +from simpleperf_utils import BaseArgumentParser, ReportLibOptions StringID = int @@ -67,6 +69,10 @@ class Sample(NamedTuple): stack_id: Optional[StackID] time_ms: Milliseconds responsiveness: int + complete_stack: bool + + def to_json(self): + return [self.stack_id, self.time_ms, self.responsiveness] # Schema: https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/profile.js#L425 @@ -119,6 +125,14 @@ CATEGORIES = [ ] +def is_complete_stack(stack: List[str]) -> bool: + """ Check if the callstack is complete. The stack starts from root. """ + for entry in stack: + if ('__libc_init' in entry) or ('__start_thread' in entry): + return True + return False + + @dataclass class Thread: """A builder for a profile of a single thread. @@ -203,7 +217,7 @@ class Thread: )) return frame_id - def _add_sample(self, comm: str, stack: List[str], time_ms: Milliseconds) -> None: + def add_sample(self, comm: str, stack: List[str], time_ms: Milliseconds) -> None: """Add a timestamped stack trace sample to the thread builder. Args: @@ -223,13 +237,43 @@ class Thread: self.samples.append(Sample(stack_id=prefix_stack_id, time_ms=time_ms, - responsiveness=0)) + responsiveness=0, + complete_stack=is_complete_stack(stack))) - def _to_json_dict(self) -> Dict: - """Converts this Thread to GeckoThread JSON format.""" - # The samples aren't guaranteed to be in order. Sort them by time. + def sort_samples(self) -> None: + """ The samples aren't guaranteed to be in order. Sort them by time. """ self.samples.sort(key=lambda s: s.time_ms) + def remove_stack_gaps(self, max_remove_gap_length: int, gap_distr: Dict[int, int]) -> None: + """ Ideally all callstacks are complete. But some may be broken for different reasons. + To create a smooth view in "Stack Chart", remove small gaps of broken callstacks. + + Args: + max_remove_gap_length: the max length of continuous broken-stack samples to remove + """ + if max_remove_gap_length == 0: + return + i = 0 + remove_flags = [False] * len(self.samples) + while i < len(self.samples): + if self.samples[i].complete_stack: + i += 1 + continue + n = 1 + while (i + n < len(self.samples)) and (not self.samples[i + n].complete_stack): + n += 1 + gap_distr[n] += 1 + if n <= max_remove_gap_length: + for j in range(i, i + n): + remove_flags[j] = True + i += n + if True in remove_flags: + old_samples = self.samples + self.samples = [s for s, remove in zip(old_samples, remove_flags) if not remove] + + def to_json_dict(self) -> Dict: + """Converts this Thread to GeckoThread JSON format.""" + # Gecko profile format is row-oriented data as List[List], # And a schema for interpreting each index. # Schema: @@ -258,7 +302,7 @@ class Thread: "time": 1, "responsiveness": 2, }, - "data": self.samples + "data": [s.to_json() for s in self.samples], }, # https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L156 "frameTable": { @@ -291,11 +335,37 @@ class Thread: } +def remove_stack_gaps(max_remove_gap_length: int, thread_map: Dict[int, Thread]) -> None: + """ Remove stack gaps for each thread, and print status. """ + if max_remove_gap_length == 0: + return + total_sample_count = 0 + remove_sample_count = 0 + gap_distr = Counter() + for tid in list(thread_map.keys()): + thread = thread_map[tid] + old_n = len(thread.samples) + thread.remove_stack_gaps(max_remove_gap_length, gap_distr) + new_n = len(thread.samples) + total_sample_count += old_n + remove_sample_count += old_n - new_n + if new_n == 0: + del thread_map[tid] + if total_sample_count != 0: + logging.info('Remove stack gaps with length <= %d. %d (%.2f%%) samples are removed.', + max_remove_gap_length, remove_sample_count, + remove_sample_count / total_sample_count * 100 + ) + logging.debug('Stack gap length distribution among samples (gap_length: count): %s', + gap_distr) + + def _gecko_profile( record_file: str, symfs_dir: Optional[str], kallsyms_file: Optional[str], - report_lib_options: ReportLibOptions) -> GeckoProfile: + report_lib_options: ReportLibOptions, + max_remove_gap_length: int) -> GeckoProfile: """convert a simpleperf profile to gecko format""" lib = ReportLib() @@ -319,7 +389,6 @@ def _gecko_profile( if sample is None: lib.Close() break - event = lib.GetEventOfCurrentSample() symbol = lib.GetSymbolOfCurrentSample() callchain = lib.GetCallChainOfCurrentSample() sample_time_ms = sample.time / 1000000 @@ -336,7 +405,7 @@ def _gecko_profile( if thread is None: thread = Thread(comm=sample.thread_comm, pid=sample.pid, tid=sample.tid) threadMap[sample.tid] = thread - thread._add_sample( + thread.add_sample( comm=sample.thread_comm, stack=stack, # We are being a bit fast and loose here with time here. simpleperf @@ -347,7 +416,12 @@ def _gecko_profile( # setting `simpleperf record --clockid realtime`. time_ms=sample_time_ms) - threads = [thread._to_json_dict() for thread in threadMap.values()] + for thread in threadMap.values(): + thread.sort_samples() + + remove_stack_gaps(max_remove_gap_length, threadMap) + + threads = [thread.to_json_dict() for thread in threadMap.values()] profile_timestamp = meta_info.get('timestamp') end_time_ms = (int(profile_timestamp) * 1000) if profile_timestamp else 0 @@ -397,13 +471,22 @@ def main() -> None: parser.add_argument('--kallsyms', help='Set the path to find kernel symbols.') parser.add_argument('-i', '--record_file', nargs='?', default='perf.data', help='Default is perf.data.') + parser.add_argument('--remove-gaps', metavar='MAX_GAP_LENGTH', dest='max_remove_gap_length', + type=int, default=3, help=""" + Ideally all callstacks are complete. But some may be broken for different + reasons. To create a smooth view in "Stack Chart", remove small gaps of + broken callstacks. MAX_GAP_LENGTH is the max length of continuous + broken-stack samples we want to remove. + """ + ) parser.add_report_lib_options() args = parser.parse_args() profile = _gecko_profile( record_file=args.record_file, symfs_dir=args.symfs, kallsyms_file=args.kallsyms, - report_lib_options=args.report_lib_options) + report_lib_options=args.report_lib_options, + max_remove_gap_length=args.max_remove_gap_length) json.dump(profile, sys.stdout, sort_keys=True) diff --git a/simpleperf/scripts/report_sample.py b/simpleperf/scripts/report_sample.py index dc5c4e2b..9b6947f7 100755 --- a/simpleperf/scripts/report_sample.py +++ b/simpleperf/scripts/report_sample.py @@ -18,6 +18,7 @@ """report_sample.py: report samples in the same format as `perf script`. """ +import sys from simpleperf_report_lib import ReportLib from simpleperf_utils import BaseArgumentParser, flatten_arg_list, ReportLibOptions from typing import List, Set, Optional @@ -88,8 +89,13 @@ def main(): parser.add_argument('--show_tracing_data', action='store_true', help='print tracing data.') parser.add_argument('--header', action='store_true', help='Show metadata header, like perf script --header') + parser.add_argument('-o', '--output_file', default='', help=""" + The path of the generated report. Default is stdout.""") parser.add_report_lib_options() args = parser.parse_args() + # If the output file has been set, redirect stdout. + if args.output_file != '' and args.output_file != '-': + sys.stdout = open(file=args.output_file, mode='w') report_sample( record_file=args.record_file, symfs_dir=args.symfs, diff --git a/simpleperf/scripts/simpleperf_utils.py b/simpleperf/scripts/simpleperf_utils.py index 8e968fc4..f8d50dca 100644 --- a/simpleperf/scripts/simpleperf_utils.py +++ b/simpleperf/scripts/simpleperf_utils.py @@ -447,7 +447,7 @@ class BinaryFinder: return path # Find binary by path in binary cache. if self.binary_cache_dir: - path = self.binary_cache_dir / dso_path_in_record_file[1:] + path = self.binary_cache_dir / dso_path_in_record_file[1:].replace('/', os.sep) if self._check_path(path, expected_build_id): return path # Find binary by its absolute path. diff --git a/simpleperf/scripts/test/binary_cache_builder_test.py b/simpleperf/scripts/test/binary_cache_builder_test.py index 153665ad..ece7d6d4 100644 --- a/simpleperf/scripts/test/binary_cache_builder_test.py +++ b/simpleperf/scripts/test/binary_cache_builder_test.py @@ -19,6 +19,7 @@ import os from pathlib import Path import shutil import tempfile +import zipfile from binary_cache_builder import BinaryCacheBuilder from simpleperf_utils import ReadElf, remove, ToolFinder @@ -36,12 +37,12 @@ class TestBinaryCacheBuilder(TestBase): filename = 'simpleperf_runtest_two_functions_arm' origin_file = TestHelper.testdata_path(filename) source_file = os.path.join(symfs_dir, filename) - target_file = os.path.join('binary_cache', filename) - expected_build_id = readelf.get_build_id(origin_file) + build_id = readelf.get_build_id(origin_file) binary_cache_builder = BinaryCacheBuilder(TestHelper.ndk_path, False) - binary_cache_builder.binaries['simpleperf_runtest_two_functions_arm'] = expected_build_id + binary_cache_builder.binaries['simpleperf_runtest_two_functions_arm'] = build_id # Copy binary if target file doesn't exist. + target_file = binary_cache_builder.find_path_in_cache(filename) remove(target_file) self.run_cmd([strip, '--strip-all', '-o', source_file, origin_file]) binary_cache_builder.copy_binaries_from_symfs_dirs([symfs_dir]) @@ -62,34 +63,69 @@ class TestBinaryCacheBuilder(TestBase): binary_cache_builder.binaries['elf'] = '' symfs_dir = TestHelper.testdata_path('data/symfs_without_build_id') source_file = os.path.join(symfs_dir, 'elf') - target_file = os.path.join('binary_cache', 'elf') + target_file = binary_cache_builder.find_path_in_cache('elf') binary_cache_builder.copy_binaries_from_symfs_dirs([symfs_dir]) self.assertTrue(filecmp.cmp(target_file, source_file)) binary_cache_builder.pull_binaries_from_device() self.assertTrue(filecmp.cmp(target_file, source_file)) + def test_copy_binary_with_different_name(self): + # Build symfs_dir. + symfs_dir = self.test_dir / 'symfs_dir' + remove(symfs_dir) + symfs_dir.mkdir() + filename = 'simpleperf_runtest_two_functions_arm' + origin_file = TestHelper.testdata_path(filename) + modified_name = 'two_functions_arm' + source_file = os.path.join(symfs_dir, modified_name) + shutil.copy(origin_file, source_file) + + # Copy binary with the same build id but a different name. + builder = BinaryCacheBuilder(TestHelper.ndk_path, False) + builder.binaries[filename] = builder.readelf.get_build_id(origin_file) + builder.copy_binaries_from_symfs_dirs([symfs_dir]) + + target_file = builder.find_path_in_cache(filename) + self.assertTrue(filecmp.cmp(target_file, source_file)) + + def test_copy_binary_for_native_lib_embedded_in_apk(self): + apk_path = TestHelper.testdata_path('data/app/com.example.hellojni-1/base.apk') + symfs_dir = self.test_dir / 'symfs_dir' + with zipfile.ZipFile(apk_path, 'r') as zip_ref: + zip_ref.extractall(symfs_dir) + builder = BinaryCacheBuilder(TestHelper.ndk_path, False) + builder.collect_used_binaries( + TestHelper.testdata_path('has_embedded_native_libs_apk_perf.data')) + builder.copy_binaries_from_symfs_dirs([symfs_dir]) + + device_path = [p for p in builder.binaries if 'libhello-jni.so' in p][0] + target_file = builder.find_path_in_cache(device_path) + self.assertTrue(target_file.is_file()) + # Check that we are not using path format of embedded lib in apk. Because + # simpleperf can't use it from binary_cache. + self.assertNotIn('!/', str(target_file)) + def test_prefer_binary_with_debug_info(self): binary_cache_builder = BinaryCacheBuilder(TestHelper.ndk_path, False) binary_cache_builder.collect_used_binaries( TestHelper.testdata_path('runtest_two_functions_arm64_perf.data')) + filename = 'simpleperf_runtest_two_functions_arm64' # Create a symfs_dir, which contains elf file with and without debug info. with tempfile.TemporaryDirectory() as tmp_dir: shutil.copy( TestHelper.testdata_path( 'simpleperf_runtest_two_functions_arm64_without_debug_info'), - Path(tmp_dir) / 'simpleperf_runtest_two_functions_arm64') + Path(tmp_dir) / filename) debug_dir = Path(tmp_dir) / 'debug' debug_dir.mkdir() - shutil.copy(TestHelper.testdata_path( - 'simpleperf_runtest_two_functions_arm64'), debug_dir) + debug_file = TestHelper.testdata_path(filename) + shutil.copy(debug_file, debug_dir) # Check if the elf file with debug info is chosen. binary_cache_builder.copy_binaries_from_symfs_dirs([tmp_dir]) - elf_path = (Path(binary_cache_builder.binary_cache_dir) / 'data' / - 'local' / 'tmp' / 'simpleperf_runtest_two_functions_arm64') - self.assertTrue(elf_path.is_file()) - self.assertIn('.debug_info', binary_cache_builder.readelf.get_sections(elf_path)) + target_file = binary_cache_builder.find_path_in_cache('/data/local/tmp/' + filename) + self.assertTrue(filecmp.cmp(target_file, debug_file)) def test_create_build_id_list(self): symfs_dir = TestHelper.testdata_dir @@ -97,9 +133,10 @@ class TestBinaryCacheBuilder(TestBase): binary_cache_builder.collect_used_binaries( TestHelper.testdata_path('runtest_two_functions_arm64_perf.data')) binary_cache_builder.copy_binaries_from_symfs_dirs([symfs_dir]) - elf_path = (Path(binary_cache_builder.binary_cache_dir) / 'data' / - 'local' / 'tmp' / 'simpleperf_runtest_two_functions_arm64') - self.assertTrue(elf_path.is_file()) + + target_file = binary_cache_builder.find_path_in_cache( + '/data/local/tmp/simpleperf_runtest_two_functions_arm64') + self.assertTrue(target_file.is_file()) binary_cache_builder.create_build_id_list() build_id_list_path = Path(binary_cache_builder.binary_cache_dir) / 'build_id_list' diff --git a/simpleperf/scripts/test/gecko_profile_generator_test.py b/simpleperf/scripts/test/gecko_profile_generator_test.py index 5a0d45ec..97cedd6f 100644 --- a/simpleperf/scripts/test/gecko_profile_generator_test.py +++ b/simpleperf/scripts/test/gecko_profile_generator_test.py @@ -18,7 +18,7 @@ import json import os import re import tempfile -from typing import List, Optional, Set +from typing import Dict, List, Optional, Set from . test_utils import TestBase, TestHelper @@ -31,8 +31,12 @@ class TestGeckoProfileGenerator(TestBase): args.extend(options) return self.run_cmd(args, return_output=True) + def generate_profile(self, testdata_file: str, options: Optional[List[str]] = None) -> Dict: + output = self.run_generator(testdata_file, options) + return json.loads(output) + def test_golden(self): - output = self.run_generator('perf_with_interpreter_frames.data') + output = self.run_generator('perf_with_interpreter_frames.data', ['--remove-gaps', '0']) got = json.loads(output) golden_path = TestHelper.testdata_path('perf_with_interpreter_frames.gecko.json') with open(golden_path) as f: @@ -43,7 +47,8 @@ class TestGeckoProfileGenerator(TestBase): def test_sample_filters(self): def get_threads_for_filter(filter: str) -> Set[int]: - report = self.run_generator('perf_display_bitmaps.data', filter.split()) + report = self.run_generator('perf_display_bitmaps.data', + filter.split() + ['--remove-gaps', '0']) pattern = re.compile(r'"tid":\s+(\d+),') threads = set() for m in re.finditer(pattern, report): @@ -79,3 +84,17 @@ class TestGeckoProfileGenerator(TestBase): self.assertNotIn(art_frame_str, report) report = self.run_generator('perf_with_interpreter_frames.data', ['--show-art-frames']) self.assertIn(art_frame_str, report) + + def test_remove_gaps(self): + testdata = 'perf_with_interpreter_frames.data' + + def get_sample_count(options: Optional[List[str]] = None) -> int: + data = self.generate_profile(testdata, options) + sample_count = 0 + for thread in data['threads']: + sample_count += len(thread['samples']['data']) + return sample_count + # By default, the gap sample is removed. + self.assertEqual(4031, get_sample_count()) + # Use `--remove-gaps 0` to disable removing gaps. + self.assertEqual(4032, get_sample_count(['--remove-gaps', '0'])) diff --git a/simpleperf/scripts/test/report_sample_test.py b/simpleperf/scripts/test/report_sample_test.py index 5b6681cb..83f1d36b 100644 --- a/simpleperf/scripts/test/report_sample_test.py +++ b/simpleperf/scripts/test/report_sample_test.py @@ -20,6 +20,7 @@ import tempfile from typing import List, Optional, Set from . test_utils import TestBase, TestHelper +from simpleperf_utils import remove class TestReportSample(TestBase): @@ -36,6 +37,33 @@ class TestReportSample(TestBase): want = f.read() self.assertEqual(got, want) + def test_output_flag(self): + # Test short form flag. + remove('some.output') + got = self.get_record_data_string('perf_display_bitmaps.data', + ['-o', 'some.output']) + self.assertEqual(got, '') + self.check_exist(filename='some.output') + + # Test long form flag + remove("some.output") + got = self.get_record_data_string('perf_display_bitmaps.data', + ['--output_file', 'some.output']) + self.assertEqual(got, '') + self.check_exist(filename="some.output") + + # Verify that the output file contains expected data + with open(TestHelper.testdata_path('perf_display_bitmaps.perf-script')) as f: + want = f.read() + with open('some.output') as f: + got = f.read() + self.assertEqual(got, want) + + # Verify that an output spec of '-' is stdout + remove("some.output") + got = self.get_record_data_string('perf_display_bitmaps.data', ['-o', '-']) + self.assertEqual(got, want) + def test_comm_filter_to_renderthread(self): got = self.get_record_data_string('perf_display_bitmaps.data', ['--comm', 'RenderThread']) self.assertIn('RenderThread', got) diff --git a/simpleperf/scripts/test/tools_test.py b/simpleperf/scripts/test/tools_test.py index 86e250c7..9c9fbd7e 100644 --- a/simpleperf/scripts/test/tools_test.py +++ b/simpleperf/scripts/test/tools_test.py @@ -341,18 +341,22 @@ system/extras/simpleperf/runtest/two_functions.cpp:21:3 build_id = readelf.get_build_id(elf_path) self.assertGreater(len(build_id), 0) binary_cache_builder.binaries[elf_name] = build_id + + filename_without_build_id = '/data/symfs_without_build_id/elf' + binary_cache_builder.binaries[filename_without_build_id] = '' + binary_cache_builder.copy_binaries_from_symfs_dirs([TestHelper.testdata_dir]) binary_cache_builder.create_build_id_list() # Test BinaryFinder. - path_in_binary_cache = Path(binary_cache_builder.binary_cache_dir, elf_name) + path_in_binary_cache = binary_cache_builder.find_path_in_cache(elf_name) binary_finder = BinaryFinder(binary_cache_builder.binary_cache_dir, readelf) # Find binary using build id. path = binary_finder.find_binary('[not_exist_file]', build_id) self.assertEqual(path, path_in_binary_cache) # Find binary using path. - path = binary_finder.find_binary('/' + elf_name, None) - self.assertEqual(path, path_in_binary_cache) + path = binary_finder.find_binary(filename_without_build_id, None) + self.assertIsNotNone(path) # Find binary using absolute path. path = binary_finder.find_binary(str(path_in_binary_cache), None) self.assertEqual(path, path_in_binary_cache) diff --git a/simpleperf/utils.cpp b/simpleperf/utils.cpp index a0418167..6357d1ea 100644 --- a/simpleperf/utils.cpp +++ b/simpleperf/utils.cpp @@ -40,6 +40,9 @@ #include <Xz.h> #include <XzCrc64.h> +#include "RegEx.h" +#include "environment.h" + namespace simpleperf { using android::base::ParseInt; @@ -424,6 +427,54 @@ std::optional<std::set<pid_t>> GetTidsFromString(const std::string& s, bool chec return tids; } +std::optional<std::set<pid_t>> GetPidsFromStrings(const std::vector<std::string>& strs, + bool check_if_exists, + bool support_progress_name_regex) { + std::set<pid_t> pids; + std::vector<std::unique_ptr<RegEx>> regs; + for (const auto& s : strs) { + for (const auto& p : Split(s, ",")) { + int pid; + if (ParseInt(p.c_str(), &pid, 0)) { + if (check_if_exists && !IsDir(StringPrintf("/proc/%d", pid))) { + LOG(ERROR) << "no process with pid " << pid; + return std::nullopt; + } + pids.insert(pid); + } else if (support_progress_name_regex) { + auto reg = RegEx::Create(p); + if (!reg) { + return std::nullopt; + } + regs.emplace_back(std::move(reg)); + } else { + LOG(ERROR) << "invalid pid: " << p; + return std::nullopt; + } + } + } + if (!regs.empty()) { +#if defined(__linux__) + for (pid_t pid : GetAllProcesses()) { + std::string process_name = GetCompleteProcessName(pid); + if (process_name.empty()) { + continue; + } + for (const auto& reg : regs) { + if (reg->Search(process_name)) { + pids.insert(pid); + break; + } + } + } +#else // defined(__linux__) + LOG(ERROR) << "progress name regex isn't supported"; + return std::nullopt; +#endif // defined(__linux__) + } + return pids; +} + size_t SafeStrlen(const char* s, const char* end) { const char* p = s; while (p < end && *p != '\0') { @@ -432,4 +483,12 @@ size_t SafeStrlen(const char* s, const char* end) { return p - s; } +OverflowResult SafeAdd(uint64_t a, uint64_t b) { + OverflowResult result; + if (__builtin_add_overflow(a, b, &result.value)) { + result.overflow = true; + } + return result; +} + } // namespace simpleperf diff --git a/simpleperf/utils.h b/simpleperf/utils.h index ebc96068..cd169889 100644 --- a/simpleperf/utils.h +++ b/simpleperf/utils.h @@ -252,6 +252,9 @@ std::string GetSimpleperfVersion(); std::optional<std::set<int>> GetCpusFromString(const std::string& s); std::optional<std::set<pid_t>> GetTidsFromString(const std::string& s, bool check_if_exists); +std::optional<std::set<pid_t>> GetPidsFromStrings(const std::vector<std::string>& strs, + bool check_if_exists, + bool support_progress_name_regex); template <typename T> std::optional<std::set<T>> ParseUintVector(const std::string& s) { @@ -275,6 +278,13 @@ static inline void HashCombine(size_t& seed, const T& val) { size_t SafeStrlen(const char* s, const char* end); +struct OverflowResult { + bool overflow = false; + uint64_t value = 0; +}; + +OverflowResult SafeAdd(uint64_t a, uint64_t b); + } // namespace simpleperf #endif // SIMPLE_PERF_UTILS_H_ diff --git a/simpleperf/utils_test.cpp b/simpleperf/utils_test.cpp index 725ae4db..2dfbd901 100644 --- a/simpleperf/utils_test.cpp +++ b/simpleperf/utils_test.cpp @@ -14,12 +14,14 @@ * limitations under the License. */ +#include "utils.h" + #include <gtest/gtest.h> #include <android-base/file.h> +#include "environment.h" #include "get_test_data.h" -#include "utils.h" using namespace simpleperf; @@ -75,6 +77,20 @@ TEST(utils, GetTidsFromString) { ASSERT_EQ(GetTidsFromString("-2", false), std::nullopt); } +TEST(utils, GetPidsFromStrings) { + ASSERT_EQ(GetPidsFromStrings({"0,12", "9"}, false, false), + std::make_optional(std::set<pid_t>({0, 9, 12}))); + ASSERT_EQ(GetPidsFromStrings({"-2"}, false, false), std::nullopt); +#if defined(__linux__) + pid_t pid = getpid(); + ASSERT_EQ(GetPidsFromStrings({std::to_string(pid)}, true, false), + std::make_optional(std::set<pid_t>({pid}))); + std::string process_name = GetCompleteProcessName(pid); + ASSERT_EQ(GetPidsFromStrings({process_name}, true, true), + std::make_optional(std::set<pid_t>({pid}))); +#endif // defined(__linux__) +} + TEST(utils, LineReader) { TemporaryFile tmpfile; close(tmpfile.release()); |