diff options
author | Android Build Coastguard Worker <android-build-coastguard-worker@google.com> | 2024-03-12 23:54:17 +0000 |
---|---|---|
committer | Gerrit Code Review <noreply-gerritcodereview@google.com> | 2024-03-12 23:54:17 +0000 |
commit | f5a14a11a6146cb9c3e2801737c23681478b607d (patch) | |
tree | 61ae2e0c9a640a74304892130b7c5c764a5c8102 | |
parent | 3b822f611afbbcc1e62c3110daff07898c23fc34 (diff) | |
parent | f36506d482269ceb2533075a47ba2e72c6497cb4 (diff) | |
download | extras-f5a14a11a6146cb9c3e2801737c23681478b607d.tar.gz |
Merge "Snap for 11566117 from 0a775d8aead45c06dc1d2238901d5f4dc4089258 to sdk-release" into sdk-releaseplatform-tools-35.0.1
-rw-r--r-- | kcmdlinectrl/kcmdlinectrl.cc | 28 | ||||
-rw-r--r--[-rwxr-xr-x] | pagecache/pagecache.py | 5 | ||||
-rw-r--r-- | pinner/Android.bp | 55 | ||||
-rw-r--r-- | pinner/OWNERS | 2 | ||||
-rw-r--r-- | pinner/README.md | 144 | ||||
-rw-r--r-- | pinner/include/meminspect.h | 268 | ||||
-rw-r--r-- | pinner/include/pin_utils.h | 116 | ||||
-rw-r--r-- | pinner/meminspect.cpp | 324 | ||||
-rw-r--r-- | pinner/pin_utils.cpp | 347 | ||||
-rw-r--r-- | pinner/pintool.cpp | 343 | ||||
-rw-r--r-- | pinner/sample_pinconfig.txt | 5 | ||||
-rw-r--r-- | pinner/tests/Android.bp | 95 | ||||
-rw-r--r-- | pinner/tests/TEST_MAPPING | 10 | ||||
-rw-r--r-- | pinner/tests/meminspect_tests.cpp | 284 | ||||
-rw-r--r-- | pinner/tests/pintool_tests.cpp | 137 | ||||
-rw-r--r-- | profcollectd/libprofcollectd/config.rs | 3 |
16 files changed, 2148 insertions, 18 deletions
diff --git a/kcmdlinectrl/kcmdlinectrl.cc b/kcmdlinectrl/kcmdlinectrl.cc index 5dd55af6..aafdb9a8 100644 --- a/kcmdlinectrl/kcmdlinectrl.cc +++ b/kcmdlinectrl/kcmdlinectrl.cc @@ -27,12 +27,10 @@ #include <functional> #include <iostream> -using namespace std; - void PrintUsage(const char* progname) { - std::cerr << "USAGE: " << progname << " get [PROPERTY]" << endl; - std::cerr << " " << progname << " store [PROPERTY] [VALUE]" << endl; - std::cerr << " " << progname << " update-props" << endl; + std::cerr << "USAGE: " << progname << " get [PROPERTY]" << std::endl; + std::cerr << " " << progname << " store [PROPERTY] [VALUE]" << std::endl; + std::cerr << " " << progname << " update-props" << std::endl; } int UpdateProps() { @@ -40,7 +38,7 @@ int UpdateProps() { .magic = MISC_KCMDLINE_MAGIC_HEADER}; std::string err; if (!ReadMiscKcmdlineMessage(&m, &err)) { - LOG(ERROR) << "Failed to read from misc: " << err << endl; + LOG(ERROR) << "Failed to read from misc: " << err; return 1; } @@ -64,12 +62,12 @@ int PrintProperty(const char* property_name) { std::string err; if (!ReadMiscKcmdlineMessage(&m, &err)) { - LOG(ERROR) << "Failed to read from misc: " << err << endl; + LOG(ERROR) << "Failed to read from misc: " << err; return 1; } if (m.magic != MISC_KCMDLINE_MAGIC_HEADER || m.version != MISC_KCMDLINE_MESSAGE_VERSION) { - cout << "kcmdline message is invalid, treating all flags as zero" << endl; + std::cout << "kcmdline message is invalid, treating all flags as zero" << std::endl; m = {.version = MISC_KCMDLINE_MESSAGE_VERSION, .magic = MISC_KCMDLINE_MAGIC_HEADER, .kcmdline_flags = 0}; @@ -78,10 +76,10 @@ int PrintProperty(const char* property_name) { if (!strcmp(property_name, "binder")) { bool use_rust_binder = (m.kcmdline_flags & MISC_KCMDLINE_BINDER_RUST) != 0; const char* binder_value = use_rust_binder ? "rust" : "c"; - cout << "binder=" << binder_value << endl; + std::cout << "binder=" << binder_value << std::endl; return 0; } else { - LOG(ERROR) << "Unknown property name: " << property_name << endl; + LOG(ERROR) << "Unknown property name: " << property_name; return 1; } } @@ -92,12 +90,12 @@ int StoreProperty(const char* property_name, const char* new_value) { std::string err; if (!ReadMiscKcmdlineMessage(&m, &err)) { - LOG(ERROR) << "Failed to read from misc: " << err << endl; + LOG(ERROR) << "Failed to read from misc: " << err; return 1; } if (m.magic != MISC_KCMDLINE_MAGIC_HEADER || m.version != MISC_KCMDLINE_MESSAGE_VERSION) { - cout << "kcmdline message is invalid, resetting it" << endl; + std::cout << "kcmdline message is invalid, resetting it" << std::endl; m = {.version = MISC_KCMDLINE_MESSAGE_VERSION, .magic = MISC_KCMDLINE_MAGIC_HEADER, .kcmdline_flags = 0}; @@ -109,16 +107,16 @@ int StoreProperty(const char* property_name, const char* new_value) { } else if (!strcmp(new_value, "c")) { m.kcmdline_flags &= !MISC_KCMDLINE_BINDER_RUST; } else { - LOG(ERROR) << "Binder property can only by 'c' or 'rust', but got " << new_value << endl; + LOG(ERROR) << "Binder property can only be 'c' or 'rust', but got " << new_value; return 1; } } else { - LOG(ERROR) << "Unknown property name: " << property_name << endl; + LOG(ERROR) << "Unknown property name: " << property_name; return 1; } if (!WriteMiscKcmdlineMessage(m, &err)) { - LOG(ERROR) << "Failed to write to misc: " << err << endl; + LOG(ERROR) << "Failed to write to misc: " << err; return 1; } diff --git a/pagecache/pagecache.py b/pagecache/pagecache.py index cb9c7396..3f96a5d1 100755..100644 --- a/pagecache/pagecache.py +++ b/pagecache/pagecache.py @@ -261,8 +261,9 @@ def get_inode_data(datafile, dumpfile, adb_serial): else: # Build inode maps if we were tracing page cache print('Downloading inode data from device') - stat_dump = AdbUtils.do_preprocess_adb_cmd('find /system /data /vendor ' + - '-exec stat -c "%d %i %s %n" {} \;', adb_serial) + stat_dump = AdbUtils.do_preprocess_adb_cmd( + 'find /apex /system /system_ext /product /data /vendor ' + + '-exec stat -c "%d %i %s %n" {} \;', adb_serial) if stat_dump is None: print 'Could not retrieve inode data from device.' sys.exit(1) diff --git a/pinner/Android.bp b/pinner/Android.bp new file mode 100644 index 00000000..40197ba2 --- /dev/null +++ b/pinner/Android.bp @@ -0,0 +1,55 @@ +// +// Copyright (C) 2017 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +cc_library_static { + name: "libmeminspect", + srcs: [ + "meminspect.cpp", + "pin_utils.cpp" + ], + shared_libs: [ + "libbase", + "libziparchive", + ], + export_shared_lib_headers: ["libziparchive"], + export_include_dirs: ["include"], + cppflags: [ + "-g", + "-Wall", + "-Werror", + ], +} + +cc_binary { + name: "pintool", + srcs: ["pintool.cpp"], + shared_libs: [ + "libbase", + "libziparchive", + ], + static_libs: [ + "libmeminspect", + ], + cppflags: [ + "-g", + "-Wall", + "-Werror", + ], +} diff --git a/pinner/OWNERS b/pinner/OWNERS new file mode 100644 index 00000000..0636f908 --- /dev/null +++ b/pinner/OWNERS @@ -0,0 +1,2 @@ +edgararriaga@google.com +shayba@google.com
\ No newline at end of file diff --git a/pinner/README.md b/pinner/README.md new file mode 100644 index 00000000..2115e809 --- /dev/null +++ b/pinner/README.md @@ -0,0 +1,144 @@ +# Pin Tool + +This tool is currently used mainly for: + +1) Inspecting resident memory and locality +2) Generating and inspecting pinlist.meta used by Android's PinnerService + +For memory inspection, it allows probing live memory and providing the memory +locations that are resident which can be used for diagnosis or as part of PGO +optimizations. + +## Build and Install the tool + +To build and push the binary to device +``` +mm pintool +and push $ANDROID_REPO/out/target/product/<lunchtarget>/system/bin/pintool /data/local/tmp/pintool +adb shell +cd /data/local/tmp/pintool +``` + + +## How to use the tool to generate pinner files + +Here are some sample use cases for this tool. + +### Sample Use Case 1: Probe Resident Memory for a mapped file or library and dump to console + +Executing the following command and providing the path to a library mapped by a process +it will dump the resident memory ranges. + +``` +./pintool file <path_to_your_library> --gen-probe --dump +``` + +Note: you can use any kind of mapped file such as .so, .apk, etc. + +### Sample Use Case 2: Probe per-file Resident Memory for a mapped apk file and dump to console + +Executing this command will inspect resident memory for a zip file and dump +per-file breakdowns. + +``` +./pintool file <path_to_myfile.apk> --zip --gen-probe --dump +``` + +### Sample Use Case 3: Probe per-file resident memory for a mapped apk, dump and generate pinlist.meta +``` +./pintool file <path_to_myfile.apk> --zip --gen-probe --dump -o pinlist.meta +``` + +### Sample Use Case 4: Probe per-file resident memory and filter it with a provided pinconfig +``` +./pintool file <path_to_myfile.apk> --zip --gen-probe --pinconfig pinconfig.txt --dump -o pinlist.meta +``` + +### Sample Use Case 5: Dump contents of a provided pinlist.meta file +``` +./pintool pinlist pinlist.meta --dump -v +``` + +### Sample Use Case 6: Read a zip and filter based on a pinconfig file and generate pinlist.meta without probing + +This will skip doing any probing and it will just apply filtering based on the pinconfig.txt, this is helpful +in cases where you do not intend to do any kind of PGO probing and know exactly what ranges you want to pin within your file + +``` +./pintool file <path_to_myfile.apk> --zip --pinconfig pinconfig.txt --dump -o pinlist.meta +``` + +### Sample Use Case 7: Load an existing zip probe and inspect its per-file contents + +``` +./pintool file /data/app/~~tmTrs5_XINwbpYWroRu5rA==/org.chromium.trichromelibrary_602300034-EFoOwMgVNBbwkMnp9zcWbg==/base.apk --zip --use-probe pinlist.meta --dump +``` + + +## Pinconfig File Structure + +Pinconfig files specify a custom filter to be applied on top of a generated or provided memory probe +it should specify a subset of files and optionally ranges within those files to be matched against +and subsequently kept in case a pinlist.meta is generated. + +A `pinconfig.txt` is just a list of files with a key value pair separated by a newline. + +`pinconfig.txt` structure pattern: +``` +(file <file> +[offset <value> +length <value>])* +``` +where: +<file> + Filename as a string, the parser will do a contains like operation (e.g. GLOB(*<file>*)) to match against files + within the zip file and stop on first match. +<value> + Unsigned integer value + +Note: `offset` and `length` tokens are optional and if ommited, the whole file will be considered desired. + + +Example `pinconfig.txt`: +``` +file lib/arm64-v8a/libcrashpad_handler_trampoline.so +file libmonochrome_64.so +offset 1000 +len 50000 +file libarcore_sdk_c.so +``` + +## Pinlist.meta files + +"pinlist.meta" files are consumed by PinnerService. + +These files specify a list of memory ranges to be pinned (mlocked). +If Android's PinnerService allows your app pinning, it will read the pinlist.meta +file from inside your apk's assets folder (assets/pinlist.meta) and pin based +on the specified ranges. + +Note: The PinnerService will need to support pinning your apk in order for the +pinlist.meta file to be used. + +A pinlist.meta file is a binary file with a set of tuples of OFFSET and LENGTH +stored in Big Endian format. + +4 byte: OFFSET +4 byte: LEN + +pinlist.meta +``` +OFFSET LEN* +``` + +So to read those files, it is usually helpful to use the `pintool`. + +## Other potential uses + +Outside of pinner service, the tool can be used to inspect resident memory for +any file in memory. + +## Extra information + +the pinlist.meta depends on the apk contents and needs to be regenrated if +you are pushing a new version of your apk.
\ No newline at end of file diff --git a/pinner/include/meminspect.h b/pinner/include/meminspect.h new file mode 100644 index 00000000..a2e09ec2 --- /dev/null +++ b/pinner/include/meminspect.h @@ -0,0 +1,268 @@ +#pragma once + +#include <android-base/stringprintf.h> +#include <fcntl.h> +#include <sys/endian.h> +#include <sys/mman.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <unistd.h> +#include <iostream> +#include <string> +#include <vector> +#include "ziparchive/zip_archive.h" + +#define MEMINSPECT_FAIL_OPEN 1 +#define MEMINSPECT_FAIL_FSTAT 2 +#define MEMINSPECT_FAIL_MINCORE 3 + +#define DEFAULT_PAGES_PER_MINCORE 1 + +/** + * This class stores an offset defined vma which exists + * relative to another memory address. + */ +class VmaRange { + public: + uint32_t offset; + uint32_t length; + + VmaRange() {} + VmaRange(uint32_t off, uint32_t len) : offset(off), length(len) {} + + bool is_empty() const; + + /** + * @brief Compute the intersection of this range with another range + * + * Intersection Operation: + * + * Example 1: + * [ Range A ] + * [ Range B ] + * Intersection: + * [ C ] + * + * Example 2: + * [ Range A ] [ Range B ] + * No Intersection + * + * @param target range to test against + * @return the intersection range, if none is found, empty range is returned. + */ + VmaRange intersect(const VmaRange& target) const; + + /** + * @brief Merges the current range with a target range using a union operation + * that is only successful when overlapping ranges occur. + * A visual explanation can be seen as: + * + * Union-merge Operation: + * + * Example 1: + * [ Range A ] + * [ Range B ] + * Merged: + * [ Range C ] + * + * Example 2: + * [ Range A ] [ Range B ] + * Fails, no merge available. + * + * @param target The range to test against. + * @param result Upon successfully merging, contains the resulting range. + * @return the merged range, if none is found, empty range is returned. + */ + VmaRange union_merge(const VmaRange& target) const; + + uint32_t end_offset() const; +}; + +/** + * Represents a set of memory ranges + */ +struct VmaRangeGroup { + std::vector<VmaRange> ranges; + + /** + * Compute intersection coverage between |range| and |this->ranges| + * and append it to |out_memres| + */ + void compute_coverage(const VmaRange& range, VmaRangeGroup& out_memres) const; + + /** + * Apply an offset to all existing |ranges|. + */ + void apply_offset(uint64_t offset); + + /** + * Computes total resident bytes from existing set of memory ranges. + */ + uint64_t compute_total_size(); +}; + +/** + * Represents useful immutable metadata for zip entry + */ +struct ZipEntryInfo { + std::string name; + uint64_t offset_in_zip; + uint64_t file_size_bytes; + uint64_t uncompressed_size; +}; + +/** + * Represents the resident memory coverage for a zip entry within a zip file. + */ +struct ZipEntryCoverage { + ZipEntryInfo info; + + /** + * Contains all the coverage ranges if any have been computed with |compute_coverage| + * and their offsets will be the absolute global offset from the zip file start. + */ + VmaRangeGroup coverage; + + /** + * Computes the intersection coverage for the current zip file entry + * resident memory against a provided |probe| representing another set + * of ranges. + */ + ZipEntryCoverage compute_coverage(const VmaRangeGroup& probe) const; +}; + +// Class used for inspecting resident memory for entries within a zip file +class ZipMemInspector { + /** + * Stored probe of resident ranges either computed or provided by user. + */ + VmaRangeGroup* probe_resident_ = nullptr; + + /** + * List of file entries within zip file. + */ + std::vector<ZipEntryInfo> entry_infos_; + + /** + * Path to zip file. + */ + std::string filename_; + + /** + * Result of computing coverage operations. + */ + std::vector<ZipEntryCoverage> entry_coverages_; + + /** + * Handle that allows reading the zip entries. + */ + ZipArchiveHandle handle_; + + public: + ZipMemInspector(std::string filename) : filename_(filename) {} + ~ZipMemInspector(); + + /** + * Reads zip file and computes resident memory coverage per zip entry if + * a probe is provided, if no probe is provided, then whole file coverage + * will be assumed. + * + * Note: If any zip entries have been manually added via |add_file_info| + * then coverage will be only computed against manually added entries. + * + * @return 0 on success and 1 on error + */ + int compute_per_file_coverage(); + + /** + * Computes resident memory for the entire zip file. + * + * @return 0 on success, 1 on failure + */ + int probe_resident(); + + /** + * Retrieves the currently set probe if any exists. + */ + VmaRangeGroup* get_probe(); + + /** + * Sets probe data in case you decide to pass a previously taken probe instead of a live taken + * one. + */ + void set_existing_probe(VmaRangeGroup* probe); + + /** + * Returns the result of memory coverage of each file if any has been computed via + * |compute_per_file_coverage|. + */ + std::vector<ZipEntryCoverage>& get_file_coverages(); + + /** + * Returns the file information for each zip entry. + */ + std::vector<ZipEntryInfo>& get_file_infos(); + + /** + * Add a zip entry manually. + * + * Note: Zip entries are usually retrieved by reading the |filename_| so + * this method is mostly used for cases where client wants control of + * zip file reading or for testing. + */ + void add_file_info(ZipEntryInfo& file); + + /** + * Computes the intersection coverage between provided |files| and |probe|. + * + * @return result of coverage computation + */ + static std::vector<ZipEntryCoverage> compute_coverage( + const std::vector<ZipEntryCoverage>& files, VmaRangeGroup* probe); + + private: + /** + * Read files and zip relative offsets for them. + * + * @return 0 on success, 1 on failure. + */ + int read_files_and_offsets(); +}; + +/** + * Retrieve file size in bytes for |file| + * + * @return positive value with file size on success, otherwise, returns -1 on error. + */ +int64_t get_file_size(const std::string& file); + +/** + * @brief Probe resident memory for a currently opened file in the system. + * + * @param probed_file File to probe as defined by its path. + * @param out_resident_mem Inspection result. This is populated when called. + * @param pages_per_mincore Size of mincore window used, bigger means more memory used + * during operation but slightly faster. + * @return 0 on success or on failure a non-zero error code from the following list: + * MEMINSPECT_FAIL_OPEN, MEMINSPECT_FAIL_FSTAT, MEMINSPECT_FAIL_MINCORE + */ +int probe_resident_memory(std::string probed_file, VmaRangeGroup& out_resident_mem, + int pages_per_mincore = DEFAULT_PAGES_PER_MINCORE); + +/** + * @brief Align vma ranges to a certain page size + * + * @param ranges vma ranges that have to be aligned + * @param alignment Desired alignment, this is usually the page size. + */ +void align_ranges(std::vector<VmaRange>& ranges, unsigned int alignment); + +/** + * @brief Merges a list of ranges following a union-like merge which + * means that two ranges that overlap will avoid double accounting for + * overlaps. + * + * @param ranges vma ranges that need to be merged. + * @return new vector with ranges merged. + */ +std::vector<VmaRange> merge_ranges(const std::vector<VmaRange>& ranges);
\ No newline at end of file diff --git a/pinner/include/pin_utils.h b/pinner/include/pin_utils.h new file mode 100644 index 00000000..b6066d63 --- /dev/null +++ b/pinner/include/pin_utils.h @@ -0,0 +1,116 @@ +#pragma once + +#include <list> +#include "meminspect.h" + +struct PinConfigFile { + std::string filename; + + // File relative offsets + std::vector<VmaRange> ranges; + + ZipEntryCoverage to_zipfilemem(const ZipEntryInfo& info); +}; + +struct PinConfig { + std::list<PinConfigFile> files_; + + int parse(std::string filename, bool verbose = false); +}; + +/** + * @brief Generate a pinlist file from a given list of vmas containing a list of 4-byte pairs + * representing (4-byte offset, 4-byte len) contiguous in memory and they are stored in big endian + * format. + * + * @param output_file Output file to write pinlist + * @param vmas_to_pin Set of vmas to write into pinlist file. + * @param write_quota Specifies a maximum amount o bytes to be written to the pinlist file + * or -1 means no limit. + * @return 0 on success, non-zero on failure + */ +int write_pinlist_file(const std::string& output_file, const std::vector<VmaRange>& vmas_to_pin, + int64_t write_quota = -1); + +/** + * @brief This method is the counter part of @see write_pinlist_file(). It will read an existing + * pinlist file. + * + * @param pinner_file Input pinlist file + * @param pinranges Vmas read from pinlist file. This is populated on call. + * @return 0 on success, non-zero on failure + */ +int read_pinlist_file(const std::string& pinner_file, /*out*/ std::vector<VmaRange>& pinranges); + +enum ProbeType { + UNSET, // No probe setup + GENERATE, // Generate a probe + CUSTOM // User generated probe +}; + +class PinTool { + public: + enum DumpType { PROBE, FILE_COVERAGE, FILTERED }; + + private: + std::string input_file_; + std::string custom_probe_file_; + PinConfig* pinconfig_; + std::vector<ZipEntryCoverage> filtered_files_; + bool verbose_; + ZipMemInspector* zip_inspector_ = nullptr; + + public: + PinTool(const std::string& input_file) : input_file_(input_file) { + zip_inspector_ = new ZipMemInspector(input_file_); + } + + ~PinTool() { + delete zip_inspector_; + delete pinconfig_; + } + + void set_verbose_output(bool verbose); + + // Read |probe_file| which should be a pinlist.meta style + // file and use it as current probe. + void read_probe_from_pinlist(std::string probe_file); + + // Compute a resident memory probe for |input_file_| + int probe_resident(); + + // Compute coverage for each zip entry contained within + // |input_file_|. + // Note: It only works for zip files + void compute_zip_entry_coverages(); + + /** + * Filter coverages based on a provided pinconfig style file + * See README.md for sample structure of pinconfig file. + * + * Note: It only works for zip files, for non zip files, this will be + * a no-op. + */ + void filter_zip_entry_coverages(const std::string& pinconfig_file); + + void filter_zip_entry_coverages(PinConfig* pinconfig); + + /** + * Dumps output of existing coverages to console for |type|. + */ + void dump_coverages(DumpType type); + + /** + * Writes coverages into a pinlist.meta style file. + * + * @param write_quota Maximum bytes allowed to be written to file. + */ + void write_coverages_as_pinlist(std::string output_pinlist, int64_t write_quota = -1); + + std::vector<ZipEntryCoverage> get_filtered_zip_entries(); + + /** + * Sets a user defined inspector, currently only used for testing. + */ + void set_custom_zip_inspector(ZipMemInspector* inspector); +};
\ No newline at end of file diff --git a/pinner/meminspect.cpp b/pinner/meminspect.cpp new file mode 100644 index 00000000..84e00928 --- /dev/null +++ b/pinner/meminspect.cpp @@ -0,0 +1,324 @@ +#include "meminspect.h" +#include <android-base/unique_fd.h> +#include "ziparchive/zip_archive.h" + +using namespace std; +using namespace android::base; +using namespace ::android::base; + +const static VmaRange VMA_RANGE_EMPTY = VmaRange(0, 0); + +uint32_t VmaRange::end_offset() const { + return offset + length; +} + +uint64_t VmaRangeGroup::compute_total_size() { + uint64_t total_size = 0; + for (auto&& range : ranges) { + total_size += range.length; + } + return total_size; +} + +void VmaRangeGroup::apply_offset(uint64_t offset) { + for (auto&& range : ranges) { + range.offset += offset; + } +} + +void VmaRangeGroup::compute_coverage(const VmaRange& range, VmaRangeGroup& out_memres) const { + for (auto&& resident_range : ranges) { + VmaRange intersect_res = resident_range.intersect(range); + if (!intersect_res.is_empty()) { + out_memres.ranges.push_back(intersect_res); + } + } +} + +bool VmaRange::is_empty() const { + return length == 0; +} + +VmaRange VmaRange::intersect(const VmaRange& target) const { + // First check if the slice is outside our range + if (target.end_offset() <= this->offset) { + return VMA_RANGE_EMPTY; + } + if (target.offset >= this->end_offset()) { + return VMA_RANGE_EMPTY; + } + VmaRange result; + // the slice should now be inside the range so compute the intersection. + result.offset = std::max(target.offset, this->offset); + uint32_t res_end = std::min(target.end_offset(), end_offset()); + result.length = res_end - result.offset; + + return result; +} + +VmaRange VmaRange::union_merge(const VmaRange& target) const { + VmaRange result = intersect(target); + if (result.is_empty()) { + // Disjointed ranges, no merge. + return VMA_RANGE_EMPTY; + } + + // Since there is an intersection, merge ranges between lowest + // and highest value. + result.offset = std::min(offset, target.offset); + uint32_t res_end = std::max(target.end_offset(), end_offset()); + result.length = res_end - result.offset; + return result; +} + +void align_ranges(std::vector<VmaRange>& vmas_to_align, unsigned int alignment) { + for (auto&& vma_to_align : vmas_to_align) { + uint32_t unaligned_offset = vma_to_align.offset % alignment; + vma_to_align.offset -= unaligned_offset; + vma_to_align.length += unaligned_offset; + } +} + +bool compare_range(VmaRange& a, VmaRange& b) { + return a.offset < b.offset; +} + +std::vector<VmaRange> merge_ranges(const std::vector<VmaRange>& ranges) { + if (ranges.size() <= 1) { + // Not enough ranges to perform a merge. + return ranges; + } + + std::vector<VmaRange> to_merge_ranges = ranges; + std::vector<VmaRange> merged_ranges; + // Sort the ranges to make a slightly more efficient merging. + std::sort(to_merge_ranges.begin(), to_merge_ranges.end(), compare_range); + + // The first element will always start as-is, then start merging with subsequent elements. + merged_ranges.push_back(to_merge_ranges[0]); + for (int iMerged = 0, iTarget = 1; iTarget < to_merge_ranges.size(); ++iTarget) { + VmaRange merged = merged_ranges[iMerged].union_merge(to_merge_ranges[iTarget]); + if (!merged.is_empty()) { + // Merge was successful, swallow range. + merged_ranges[iMerged] = merged; + } else { + // Merge failed, add disjointed range. + merged_ranges.push_back(to_merge_ranges[iTarget]); + ++iMerged; + } + } + + return merged_ranges; +} + +int64_t get_file_size(const std::string& file) { + unique_fd file_ufd(open(file.c_str(), O_RDONLY)); + int fd = file_ufd.get(); + if (fd == -1) { + return -1; + } + + struct stat fstat_res; + int res = fstat(fd, &fstat_res); + if (res == -1) { + return -1; + } + + return fstat_res.st_size; +} + +int probe_resident_memory(string probed_file, + /*out*/ VmaRangeGroup& resident_ranges, int pages_per_mincore) { + unique_fd probed_file_ufd(open(probed_file.c_str(), O_RDONLY)); + int probe_fd = probed_file_ufd.get(); + if (probe_fd == -1) { + return MEMINSPECT_FAIL_OPEN; + } + + int64_t total_bytes = get_file_size(probed_file); + if (total_bytes < 0) { + return MEMINSPECT_FAIL_FSTAT; + } + + char* base_address = + (char*)mmap(0, (uint64_t)total_bytes, PROT_READ, MAP_SHARED, probe_fd, /*offset*/ 0); + + // this determines how many pages to inspect per mincore syscall + unsigned char* window = new unsigned char[pages_per_mincore]; + + unsigned int page_size = sysconf(_SC_PAGESIZE); + unsigned long bytes_inspected = 0; + + // total bytes in inspection window + unsigned long window_bytes = page_size * pages_per_mincore; + + char* window_base; + bool started_vma_range = false; + uint32_t resident_vma_start_offset = 0; + for (window_base = base_address; bytes_inspected < total_bytes; + window_base += window_bytes, bytes_inspected += window_bytes) { + int res = mincore(window_base, window_bytes, window); + if (res != 0) { + if (errno == ENOMEM) { + // Did not find page, maybe it's a hole. + continue; + } + return MEMINSPECT_FAIL_MINCORE; + } + // Inspect the provided mincore window result sequentially + // and as soon as a change in residency happens a range is + // created or finished. + for (int iWin = 0; iWin < pages_per_mincore; ++iWin) { + if ((window[iWin] & (unsigned char)1) != 0) { + // Page is resident + if (!started_vma_range) { + // End of range + started_vma_range = true; + uint32_t window_offset = iWin * page_size; + resident_vma_start_offset = window_base + window_offset - base_address; + } + } else { + // Page is not resident + if (started_vma_range) { + // Start of range + started_vma_range = false; + uint32_t window_offset = iWin * page_size; + uint32_t resident_vma_end_offset = window_base + window_offset - base_address; + uint32_t resident_len = resident_vma_end_offset - resident_vma_start_offset; + VmaRange vma_range(resident_vma_start_offset, resident_len); + resident_ranges.ranges.push_back(vma_range); + } + } + } + } + // This was the last window, so close any opened vma range + if (started_vma_range) { + started_vma_range = false; + uint32_t in_memory_vma_end = window_base - base_address; + uint32_t resident_len = in_memory_vma_end - resident_vma_start_offset; + VmaRange vma_range(resident_vma_start_offset, resident_len); + resident_ranges.ranges.push_back(vma_range); + } + + return 0; +} + +ZipMemInspector::~ZipMemInspector() { + CloseArchive(handle_); + delete probe_resident_; +} + +ZipEntryCoverage ZipEntryCoverage::compute_coverage(const VmaRangeGroup& probe) const { + ZipEntryCoverage file_coverage; + file_coverage.info = info; + + // Compute coverage for each range in file against probe which represents a set of ranges. + for (auto&& range : coverage.ranges) { + probe.compute_coverage(range, file_coverage.coverage); + } + + return file_coverage; +} + +std::vector<ZipEntryCoverage> ZipMemInspector::compute_coverage( + const std::vector<ZipEntryCoverage>& files, VmaRangeGroup* probe) { + if (probe == nullptr) { + // No probe to calculate coverage against, so coverage is zero. + return std::vector<ZipEntryCoverage>(); + } + + std::vector<ZipEntryCoverage> file_coverages; + // Find the file coverage against provided probe. + for (auto&& file : files) { + // For each file, compute coverage against the probe which represents a list of ranges. + ZipEntryCoverage file_coverage = file.compute_coverage(*probe); + file_coverages.push_back(file_coverage); + } + + return file_coverages; +} + +void ZipMemInspector::add_file_info(ZipEntryInfo& file) { + entry_infos_.push_back(file); +} + +int ZipMemInspector::compute_per_file_coverage() { + if (entry_infos_.empty()) { + // We haven't read the file information yet, so do it now. + if (read_files_and_offsets()) { + cerr << "Could not read zip entries to compute coverages." << endl; + return 1; + } + } + + // All existing files should consider their whole memory as present by default. + std::vector<ZipEntryCoverage> entry_coverages; + for (auto&& entry_info : entry_infos_) { + ZipEntryCoverage entry_coverage; + entry_coverage.info = entry_info; + VmaRange file_vma_range(entry_info.offset_in_zip, entry_info.file_size_bytes); + entry_coverage.coverage.ranges.push_back(file_vma_range); + entry_coverage.coverage.compute_total_size(); + entry_coverages.push_back(entry_coverage); + } + + if (probe_resident_ != nullptr) { + // We decided to compute coverage based on a probe + entry_coverages_ = compute_coverage(entry_coverages, probe_resident_); + } else { + // No probe means whole file coverage + entry_coverages_ = entry_coverages; + } + + return 0; +} + +VmaRangeGroup* ZipMemInspector::get_probe() { + return probe_resident_; +} + +void ZipMemInspector::set_existing_probe(VmaRangeGroup* probe) { + this->probe_resident_ = probe; +} + +std::vector<ZipEntryCoverage>& ZipMemInspector::get_file_coverages() { + return entry_coverages_; +} + +int ZipMemInspector::probe_resident() { + probe_resident_ = new VmaRangeGroup(); + int res = probe_resident_memory(filename_, *probe_resident_); + if (res != 0) { + // Failed to probe + return res; + } + + return 0; +} + +std::vector<ZipEntryInfo>& ZipMemInspector::get_file_infos() { + return entry_infos_; +} + +int ZipMemInspector::read_files_and_offsets() { + if (OpenArchive(filename_.c_str(), &handle_) < 0) { + return 1; + } + void* cookie; + int res = StartIteration(handle_, &cookie); + if (res != 0) { + return 1; + } + + ZipEntry64 entry; + string name; + while (Next(cookie, &entry, &name) == 0) { + ZipEntryInfo file; + file.name = name; + file.offset_in_zip = entry.offset; + file.file_size_bytes = entry.compressed_length; + file.uncompressed_size = entry.uncompressed_length; + entry_infos_.push_back(file); + } + return 0; +} diff --git a/pinner/pin_utils.cpp b/pinner/pin_utils.cpp new file mode 100644 index 00000000..7c746bb0 --- /dev/null +++ b/pinner/pin_utils.cpp @@ -0,0 +1,347 @@ +#include "pin_utils.h" +#include <android-base/parseint.h> +#include <algorithm> +#include <fstream> +#include <map> +#include <string_view> +#include <utility> +#include <vector> + +using namespace std; +using namespace android::base; + +int write_pinlist_file(const std::string& output_file, + const std::vector<ZipEntryCoverage>& files_to_write, int64_t write_quota) { + std::vector<VmaRange> ranges; + for (auto&& file : files_to_write) { + ranges.insert(ranges.end(), file.coverage.ranges.begin(), file.coverage.ranges.end()); + } + return write_pinlist_file(output_file, ranges, write_quota); +} + +int write_pinlist_file(const std::string& output_file, const std::vector<VmaRange>& vmas_to_write, + int64_t write_quota) { + ofstream pinlist_file(output_file); + if (pinlist_file.fail()) { + return 1; + } + int64_t total_written = 0; + unsigned int page_size = sysconf(_SC_PAGESIZE); + const bool has_quota = write_quota > 0; + bool reached_quota = false; + + // The PinnerService does not require aligned offsets, however, aligning + // allows our summary results to be accurate and avoids over-accounting + // of pinning in PinnerService. + std::vector<VmaRange> processed_vmas_to_write = vmas_to_write; + align_ranges(processed_vmas_to_write, page_size); + + // When we page-align the ranges, we may cause overlaps between ranges + // as we elongate the begin offset to match the page the previous + // range may end up overlapping the current one. + processed_vmas_to_write = merge_ranges(processed_vmas_to_write); + + for (auto&& processed_vma_to_write : processed_vmas_to_write) { + uint32_t vma_start_offset = processed_vma_to_write.offset; + uint32_t vma_length = processed_vma_to_write.length; + if (has_quota && (total_written + vma_length > write_quota)) { + // We would go beyond quota, set the maximum allowed write and exit. + vma_length = write_quota - total_written; + reached_quota = true; + } + // Transform to BigEndian as PinnerService requires that endianness for reading. + uint32_t vma_start_offset_be = htobe32(vma_start_offset); + uint32_t vma_length_be = htobe32(vma_length); + cout << "Pinlist Writing start=" << vma_start_offset << " bytes=" << vma_length << endl; + pinlist_file.write(reinterpret_cast<char*>(&vma_start_offset_be), + sizeof(vma_start_offset_be)); + if (pinlist_file.fail()) { + return 1; + } + pinlist_file.write(reinterpret_cast<char*>(&vma_length_be), sizeof(vma_length_be)); + total_written += vma_length; + if (pinlist_file.fail()) { + return 1; + } + + if (reached_quota) { + break; + } + } + return 0; +} + +int read_pinlist_file(const std::string& pinner_file, /*out*/ std::vector<VmaRange>& pinranges) { + ifstream pinlist_file(pinner_file); + if (pinlist_file.fail()) { + return 1; + } + + uint32_t vma_start; + uint32_t vma_length; + while (!pinlist_file.eof()) { + pinlist_file.read(reinterpret_cast<char*>(&vma_start), sizeof(vma_start)); + pinlist_file.read(reinterpret_cast<char*>(&vma_length), sizeof(vma_length)); + if (pinlist_file.fail()) { + return 1; + } + vma_start = betoh32(vma_start); + vma_length = betoh32(vma_length); + pinranges.push_back(VmaRange(vma_start, vma_length)); + } + + return 0; +} + +ZipEntryCoverage PinConfigFile::to_zipfilemem(const ZipEntryInfo& info) { + ZipEntryCoverage file; + file.info = info; + + if (ranges.empty()) { + cout << "No ranges found for file " << info.name << " creating entire file range" << endl; + // Any file coming from pinconfig without explicit + // ranges will be assumed to be wanted in its entirety + ranges.push_back(VmaRange(0, info.file_size_bytes)); + } + + file.coverage.ranges = ranges; + + // Offsets specified in pinconfig file are relative to the file + // so transform to zip global offsets which are used for coverage + // computations. + file.coverage.apply_offset(info.offset_in_zip); + + file.coverage.compute_total_size(); + return file; +} + +int PinConfig::parse(std::string config_file, bool verbose) { + ifstream file(config_file); + string file_in_zip; + if (verbose) { + cout << "Parsing file: " << config_file << endl; + } + string token; + file >> token; + while (!file.eof()) { + if (token == "file") { + file >> file_in_zip; + PinConfigFile pin_config_file; + pin_config_file.filename = file_in_zip; + file >> token; + while (token != "file" && !file.eof()) { + VmaRange range; + // Inner parsing loop for per file config. + if (token == "offset") { + file >> token; + android::base::ParseUint(token, &range.offset); + file >> token; + if (token != "len") { + cerr << "Malformed file, expected 'len' after offset" << endl; + return 1; + } + file >> token; + android::base::ParseUint(token, &range.length); + pin_config_file.ranges.push_back(range); + } + file >> token; + } + files_.push_back(pin_config_file); + } else { + cerr << "Unexpected token: " << token << ". Exit read" << endl; + return 1; + } + } + + if (files_.empty()) { + cerr << "Failed parsing pinconfig file, no entries found." << endl; + return 1; + } + + if (verbose) { + cout << "Finished parsing Pinconfig file" << endl; + for (auto&& pin_file : files_) { + cout << "file=" << pin_file.filename << endl; + for (auto&& range : pin_file.ranges) { + cout << "offset=" << range.offset << " bytes=" << range.length << endl; + } + } + } + + return 0; +} + +void PinTool::set_custom_zip_inspector(ZipMemInspector* inspector) { + delete zip_inspector_; + zip_inspector_ = inspector; +} + +void PinTool::set_verbose_output(bool verbose) { + verbose_ = verbose; +} + +void PinTool::read_probe_from_pinlist(std::string custom_probe_file) { + custom_probe_file_ = custom_probe_file; + VmaRangeGroup* custom_probe = new VmaRangeGroup(); + read_pinlist_file(custom_probe_file_, custom_probe->ranges); + custom_probe->compute_total_size(); + if (custom_probe->ranges.empty()) { + cerr << "Did not find any memory range in " << custom_probe_file_ << endl; + delete custom_probe; + return; + } + zip_inspector_->set_existing_probe(custom_probe); +} + +int PinTool::probe_resident() { + return zip_inspector_->probe_resident(); +} + +void PinTool::compute_zip_entry_coverages() { + zip_inspector_->compute_per_file_coverage(); + if (verbose_) { + std::vector<ZipEntryInfo> files = zip_inspector_->get_file_infos(); + for (auto&& file : files) { + cout << "file found. name=" << file.name << " offset=" << file.offset_in_zip + << " uncompressed=" << file.uncompressed_size + << " compressed=" << file.file_size_bytes << endl + << endl; + } + } +} + +void PinTool::dump_coverages(PinTool::DumpType dump_type) { + std::vector<ZipEntryCoverage>* file_coverages; + if (dump_type == PinTool::DumpType::FILTERED) { + file_coverages = &filtered_files_; + } else if (dump_type == PinTool::DumpType::FILE_COVERAGE) { + file_coverages = &(zip_inspector_->get_file_coverages()); + } else { // PinTool::DumpType::PROBE + VmaRangeGroup* probe = zip_inspector_->get_probe(); + file_coverages = new vector<ZipEntryCoverage>(); + ZipEntryCoverage file; + file.coverage = *probe; + file.info.name = input_file_; + file.info.offset_in_zip = 0; + uint64_t file_size_bytes = get_file_size(input_file_); + if (file_size_bytes == -1) { + cerr << "Failed to dump, cannot fstat file: " << input_file_ << endl; + delete file_coverages; + return; + } + file.info.file_size_bytes = file_size_bytes; + file_coverages->push_back(file); + } + + for (auto&& file : *file_coverages) { + uint64_t total_size = file.coverage.compute_total_size(); + cout << file.info.name << " size(B)=" << file.info.file_size_bytes + << " resident(B)=" << total_size + << " resident(%)=" << (double)(total_size) / file.info.file_size_bytes * 100.0 << endl; + if (verbose_) { + cout << "file_base_zip_offset=" << file.info.offset_in_zip << endl; + } + cout << "file resident ranges" << endl; + if (dump_type != DumpType::PROBE) { + for (auto&& range : file.coverage.ranges) { + // The offset in the range represents the absolute absolute offset relative to the + // zip so substract the file base offset to get the relative offset within the file + // which may be what is worth for a user to specify in pinconfig.txt files. + uint64_t offset_in_file = range.offset - file.info.offset_in_zip; + + cout << "zip_offset=" << range.offset << " file_offset=" << offset_in_file + << " total_bytes=" << range.length << endl; + } + } else { + for (auto&& range : file.coverage.ranges) { + cout << "file_offset=" << range.offset << " total_bytes=" << range.length << endl; + } + } + cout << endl; + } + cout << endl; + if (dump_type == DumpType::PROBE) { + // For other dump types we do not create memory, we reuse from class. + delete file_coverages; + } +} + +void PinTool::filter_zip_entry_coverages(const std::string& pinconfig_filename) { + if (pinconfig_filename.length() == 0) { + // Nothing to do. + return; + } + + PinConfig* pinconfig = new PinConfig(); + if (pinconfig->parse(pinconfig_filename, verbose_) > 0) { + cerr << "Failed parsing pinconfig file " << pinconfig_filename << ". Skip filtering"; + delete pinconfig; + return; + } + + filter_zip_entry_coverages(pinconfig); +} + +void PinTool::filter_zip_entry_coverages(PinConfig* pinconfig) { + pinconfig_ = pinconfig; + + // Filter based on the per file configuration. + vector<ZipEntryCoverage> file_coverages = zip_inspector_->get_file_coverages(); + vector<ZipEntryCoverage>& filtered_files = filtered_files_; + + for (auto&& file_coverage : file_coverages) { + for (auto&& pinconfig_file : pinconfig_->files_) { + // Match each zip entry against every pattern in filter file. + std::string_view file_coverage_view(file_coverage.info.name.c_str()); + std::string_view pinconfig_view(pinconfig_file.filename.c_str()); + if (file_coverage_view.find(pinconfig_view) != std::string_view::npos) { + // Now that we found a match, create a file with offsets that are global to zip file + ZipEntryCoverage file_in_config = pinconfig_file.to_zipfilemem(file_coverage.info); + if (verbose_) { + cout << "Found a match: file=" << file_coverage.info.name + << " matching filter=" << pinconfig_file.filename << endl; + for (auto&& range : file_in_config.coverage.ranges) { + cout << "zip_offset=" << range.offset << " bytes=" << range.length << endl; + } + } + ZipEntryCoverage filtered_file = + file_coverage.compute_coverage(file_in_config.coverage); + filtered_files.push_back(filtered_file); + break; + } + } + } +} + +std::vector<ZipEntryCoverage> PinTool::get_filtered_zip_entries() { + return filtered_files_; +} + +void PinTool::write_coverages_as_pinlist(std::string output_pinlist, int64_t write_quota) { + std::vector<ZipEntryCoverage>* pinlist_coverages = nullptr; + if (!filtered_files_.empty()) { + // Highest preference is writing filtered files if they exist + if (verbose_) { + cout << "Writing pinconfig filtered file coverages" << endl; + } + pinlist_coverages = &filtered_files_; + } else if (!zip_inspector_->get_file_coverages().empty()) { + // Fallback to looking for file coverage computation + pinlist_coverages = &zip_inspector_->get_file_coverages(); + if (verbose_) { + cout << "Writing regular file coverages." << endl; + } + } + if (pinlist_coverages == nullptr) { + cerr << "Failed to find coverage to write to: " << output_pinlist << endl; + return; + } + int res = write_pinlist_file(output_pinlist, *pinlist_coverages, write_quota); + if (res > 0) { + cerr << "Failed to write pin file at: " << output_pinlist << endl; + } else { + if (verbose_) { + cout << "Finished writing pin file at: " << output_pinlist << endl; + } + } +}
\ No newline at end of file diff --git a/pinner/pintool.cpp b/pinner/pintool.cpp new file mode 100644 index 00000000..9aa47b7b --- /dev/null +++ b/pinner/pintool.cpp @@ -0,0 +1,343 @@ +#include <android-base/parseint.h> +#include <fcntl.h> +#include <sys/endian.h> +#include <sys/mman.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <unistd.h> +#include <iostream> +#include <string> +#include <vector> + +#include <meminspect.h> +#include <pin_utils.h> + +using namespace std; +using namespace android::base; + +enum ToolMode { + MAPPED_FILE, // Files that are mapped in memory + PINLIST, // pinlist.meta style file + UNKNOWN +}; + +void print_pinlist_ranges(const std::vector<VmaRange>& ranges) { + cout << "--pinlist memory ranges--" << endl; + for (auto&& range : ranges) { + cout << "start=" << range.offset << " bytes=" << range.length << endl; + } +} + +void print_pinlist_summary(const std::vector<VmaRange>& ranges) { + cout << "--pinlist summary--" << endl; + uint64_t total_bytes = 0; + for (auto&& range : ranges) { + total_bytes += range.length; + } + cout << "total_bytes_to_pin=" << total_bytes << endl; +} + +int perform_file_action(const vector<string>& options) { + std::string custom_probe_file; + std::string output_file; + std::string pinconfig_file; + + bool verbose = false; + bool is_zip = false; + bool dump_results = false; + ProbeType probe_type = UNSET; + int64_t write_quota = -1; // unbounded by default + + if (options.empty()) { + cerr << "Missing filename for file action, see usage for details." << endl; + return 1; + } + + std::string input_file = options[0]; + + // Validate that the file exists. + if (get_file_size(input_file) == -1) { + cerr << "Error: Could not read file: " << input_file << endl; + return 1; + } + + if (input_file.empty()) { + cerr << "Error: Should specify an input file." << endl; + return 1; + } + + // Parse flags + for (int i = 1; i < options.size(); ++i) { + string option = options[i]; + if (option == "--gen-probe") { + if (probe_type != ProbeType::UNSET) { + cerr << "Should only specify one probe treatment. See usage for details." << endl; + return 1; + } + probe_type = ProbeType::GENERATE; + continue; + } + + if (option == "--use-probe") { + if (probe_type != ProbeType::UNSET) { + cerr << "Should only specify one probe treatment. See usage for details." << endl; + return 1; + } + probe_type = ProbeType::CUSTOM; + ++i; + custom_probe_file = options[i]; + continue; + } + if (option == "--pinconfig") { + ++i; + pinconfig_file = options[i]; + continue; + } + if (option == "-o") { + ++i; + output_file = options[i]; + continue; + } + if (option == "--quota") { + ++i; + android::base::ParseInt(options[i], &write_quota); + continue; + } + if (option == "-v") { + verbose = true; + continue; + } + if (option == "--zip") { + is_zip = true; + continue; + } + if (option == "--dump") { + dump_results = true; + continue; + } + } + + if (verbose) { + cout << "Setting output pinlist file: " << output_file.c_str() << endl; + cout << "Setting input file: " << input_file.c_str() << endl; + cout << "Setting pinconfig file: " << pinconfig_file.c_str() << endl; + cout << "Setting custom probe file: " << custom_probe_file.c_str() << endl; + cout << "Setting probe type: " << probe_type << endl; + cout << "Dump enabled: " << dump_results << endl; + cout << "Is Zip file: " << is_zip << endl; + if (write_quota != -1) { + cout << "Set Write quota: " << write_quota << endl; + } + } + + PinTool pintool(input_file); + + if (is_zip) { + pintool.set_verbose_output(verbose); + if (probe_type == ProbeType::CUSTOM) { + if (verbose) { + cout << "Using custom probe file: " << custom_probe_file << endl; + } + pintool.read_probe_from_pinlist(custom_probe_file); + } else if (probe_type == ProbeType::GENERATE) { + if (verbose) { + cout << "Generating probe" << endl; + } + int res = pintool.probe_resident(); + if (res > 0) { + cerr << "Failed to generate probe. Error Code: " << res << endl; + return 1; + } + } + pintool.compute_zip_entry_coverages(); + + if (pinconfig_file.length() > 0) { + // We have provided a pinconfig file so perform filtering + // of computed coverages based on it. + pintool.filter_zip_entry_coverages(pinconfig_file); + } + + if (dump_results) { + cout << endl << "----Unfiltered file coverages----" << endl << endl; + pintool.dump_coverages(PinTool::DumpType::FILE_COVERAGE); + + if (pinconfig_file.length() > 0) { + cout << endl << "----Filtered file coverages----" << endl << endl; + pintool.dump_coverages(PinTool::DumpType::FILTERED); + } + } + + if (output_file.length() > 0) { + pintool.write_coverages_as_pinlist(output_file, write_quota); + } + + return 0; + } else { + if (probe_type != ProbeType::GENERATE) { + cerr << "Only generating probes is supported for non-zip files, please include " + "--gen-probe on your command" + << endl; + return 1; + } + + // Generic file probing will just return resident memory and offsets + // without more contextual information. + VmaRangeGroup resident; + + int res = pintool.probe_resident(); + if (res > 0) { + cerr << "Failed to generate probe. Error Code: " << res << endl; + return 1; + } + + pintool.dump_coverages(PinTool::DumpType::PROBE); + + if (output_file.length() > 0) { + res = write_pinlist_file(output_file, resident.ranges, write_quota); + if (res > 0) { + cerr << "Failed to write pin file at: " << output_file << endl; + } else if (verbose) { + cout << "Finished writing pin file at: " << output_file << endl; + } + } + return res; + } + return 0; +} + +int perform_pinlist_action(const vector<string>& options) { + string pinner_file; + bool verbose = false; + bool dump = false; + bool summary = false; + + if (options.size() < 1) { + cerr << "Missing arguments for pinlist mode. See usage for details << endl"; + return 1; + } + pinner_file = options[0]; + for (int i = 1; i < options.size(); ++i) { + string option = options[i]; + + if (option == "-v") { + verbose = true; + } + + if (option == "--dump") { + dump = true; + } + + if (option == "--summary") { + summary = true; + } + } + + if (pinner_file.empty()) { + cerr << "Error: Pinlist file to dump is missing. Specify it with '-p <file>'" << endl; + return 1; + } + + if (verbose) { + cout << "Setting file to dump: " << pinner_file.c_str() << endl; + } + + vector<VmaRange> vma_ranges; + if (read_pinlist_file(pinner_file, vma_ranges) == 1) { + cerr << "Failed reading pinlist file" << endl; + } + + if (dump) { + print_pinlist_ranges(vma_ranges); + } + + if (summary) { + print_pinlist_summary(vma_ranges); + } + + return 0; +} + +void print_usage() { + const string usage = R"( + Expected usage: pintool <mode> <required> [option] + where: + ./pintool <MODE> + <MODE> + file <filename> [option] + [option] + --gen-probe + Generate a probe from current resident memory based on provided "file" + --use-probe <path_to_input_pinlist.meta> + Use a previously generated pinlist.meta style file as the probe to match against. + --dump + Dump output contents to console. + --zip + Treat the file as a zip/apk file required for doing per-file coverage analysis and generation. + --pinconfig <path_to_pinconfig.txt> + Filter output coverage ranges using a provided pinconfig.txt style file. See README.md for samples + on the format of that file. + -v + Enable verbose output. + + pinlist <pinlist_file> [option] + <pinlist_file> + this is the file that will be used for reading and it should follow the pinlist.meta format. + [option] + --dump + Dump <pinlist_file> contents to console output. + -v + Enable verbose output. + --summary + Summary results for the pinlist.meta file + )"; + cout << usage.c_str(); +} + +int main(int argc, char** argv) { + if (argc == 1) { + print_usage(); + return 0; + } + + if (argc < 2) { + cerr << "<mode> is missing"; + return 1; + } + + if (strcmp(argv[1], "--help") == 0) { + print_usage(); + return 0; + } + + ToolMode mode = ToolMode::UNKNOWN; + if (strcmp(argv[1], "file") == 0) { + mode = ToolMode::MAPPED_FILE; + } else if (strcmp(argv[1], "pinlist") == 0) { + mode = ToolMode::PINLIST; + } + + if (mode == ToolMode::UNKNOWN) { + cerr << "Failed to find mode: " << argv[1] << ". See usage for available modes." << endl; + return 1; + } + + vector<string> options; + for (int i = 2; i < argc; ++i) { + options.push_back(argv[i]); + } + + int res; + switch (mode) { + case ToolMode::MAPPED_FILE: + res = perform_file_action(options); + break; + case ToolMode::PINLIST: + res = perform_pinlist_action(options); + break; + case ToolMode::UNKNOWN: + cerr << "Unknown <MODE> see usage for details." << endl; + return 1; + } + + return res; +} diff --git a/pinner/sample_pinconfig.txt b/pinner/sample_pinconfig.txt new file mode 100644 index 00000000..3d9a13a6 --- /dev/null +++ b/pinner/sample_pinconfig.txt @@ -0,0 +1,5 @@ +file lib/arm64-v8a/libcrashpad_handler_trampoline.so +file libmonochrome_64.so +offset 108391596 +len 33348032 +file libarcore_sdk_c.so
\ No newline at end of file diff --git a/pinner/tests/Android.bp b/pinner/tests/Android.bp new file mode 100644 index 00000000..f7b04d95 --- /dev/null +++ b/pinner/tests/Android.bp @@ -0,0 +1,95 @@ +// Copyright (C) 2018 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +cc_test { + name: "meminspect_tests", + + test_suites: ["device-tests"], + + // Required for reading-writing files which are part of the tests. + require_root: true, + + shared_libs: [ + "libbase", + "libziparchive", + ], + + cppflags: [ + "-g", + "-Wall", + "-Werror", + ], + + static_libs: [ + "libmeminspect", + ], + + target: { + android: { + srcs: ["meminspect_tests.cpp"], + }, + }, + + cflags: [ + "-Wall", + "-Wextra", + "-Werror", + "-O0", // as some tests rely on compiler keeping code as is + ], + + compile_multilib: "first", +} + +cc_test { + name: "pintool_tests", + + test_suites: ["device-tests"], + + // Required for reading-writing files which are part of the tests. + require_root: true, + + shared_libs: [ + "libbase", + "libziparchive", + ], + + cppflags: [ + "-g", + "-Wall", + "-Werror", + ], + + static_libs: [ + "libmeminspect", + ], + + target: { + android: { + srcs: ["pintool_tests.cpp"], + }, + }, + + cflags: [ + "-Wall", + "-Wextra", + "-Werror", + "-O0", // as some tests rely on compiler keeping code as is + ], + + compile_multilib: "first", +} diff --git a/pinner/tests/TEST_MAPPING b/pinner/tests/TEST_MAPPING new file mode 100644 index 00000000..73af05f1 --- /dev/null +++ b/pinner/tests/TEST_MAPPING @@ -0,0 +1,10 @@ +{ + "postsubmit": [ + { + "name": "pintool_tests" + }, + { + "name": "meminspect_tests" + } + ] +}
\ No newline at end of file diff --git a/pinner/tests/meminspect_tests.cpp b/pinner/tests/meminspect_tests.cpp new file mode 100644 index 00000000..98c51def --- /dev/null +++ b/pinner/tests/meminspect_tests.cpp @@ -0,0 +1,284 @@ +#include <gtest/gtest.h> +#include <stdio.h> + +#include <meminspect.h> + +using namespace std; + +/** + * This test is meant to be ran by directly pushing the test binary + * into the device as using atest will not provide sufficient privileges + * to execute drop_caches command. + */ +TEST(meminspect_test, inspect_matches_resident) { + // Create test file + string test_file = "/data/local/tmp/meminspect_test"; + // If for any reason a test file already existed from a previous test, remove it. + remove(test_file.c_str()); + + int test_file_fd = open(test_file.c_str(), O_RDWR | O_CREAT, "w"); + unsigned int page_size = sysconf(_SC_PAGESIZE); + if (test_file_fd == -1) { + ADD_FAILURE() << "Failed to open test file for writing. errno: " << std::strerror(errno); + close(test_file_fd); + remove(test_file.c_str()); + return; + } + + uint8_t* page_data = new uint8_t[page_size]; + for (unsigned int i = 0; i < page_size; ++i) { + page_data[i] = i + 1; + } + int pages_to_write = 100; + for (int page = 0; page < pages_to_write; ++page) { + write(test_file_fd, page_data, page_size); + } + // fsync to ensure the data is flushed to disk. + if (fsync(test_file_fd) == -1) { + ADD_FAILURE() << "fsync failed errno: " << std::strerror(errno); + close(test_file_fd); + remove(test_file.c_str()); + return; + } + close(test_file_fd); + + // Drop the pagecache to ensure we do not have memory due to it staying there after write. + int drop_cache_fd = open("/proc/sys/vm/drop_caches", O_WRONLY); + if (drop_cache_fd == -1) { + ADD_FAILURE() << "failed opening drop caches fd errno: " << std::strerror(errno); + close(test_file_fd); + remove(test_file.c_str()); + return; + } + write(drop_cache_fd, "3", 1); + fsync(drop_cache_fd); + close(drop_cache_fd); + + // Open file and page in some memory + test_file_fd = open(test_file.c_str(), O_RDONLY, "r"); + if (test_file_fd == -1) { + ADD_FAILURE() << "Failed to open test file for reading after creation. errno: " + << std::strerror(errno); + close(test_file_fd); + remove(test_file.c_str()); + return; + } + + char* base_address = (char*)mmap(0, page_size * pages_to_write, PROT_READ, MAP_SHARED, + test_file_fd, /*offset*/ 0); + if (base_address == (char*)-1) { + ADD_FAILURE() << "Failed to mmap file for reading after creation. errno: " + << std::strerror(errno); + close(test_file_fd); + remove(test_file.c_str()); + return; + } + + VmaRangeGroup vmas_resident; + int res = probe_resident_memory(test_file, vmas_resident, 1); + EXPECT_TRUE(res == 0); + + // Probing the file without reading anything yields no resident memory + EXPECT_TRUE(vmas_resident.ranges.empty()); + + // Clear our to start fresh for next probe. + vmas_resident = VmaRangeGroup(); + + int pages_to_read = 1; + char* read_data = new char[pages_to_read]; + for (int page = 0; page < pages_to_read; ++page) { + // Read 1 byte from each page to page it in. + read_data[page] = base_address[page * page_size]; + } + res = probe_resident_memory(test_file, vmas_resident, 1); + EXPECT_TRUE(res == 0); + + // The amount of memory paged in is outside our control, but we should have some. + uint64_t resident_total_size = vmas_resident.compute_total_size(); + EXPECT_TRUE(resident_total_size > 0); + EXPECT_TRUE(vmas_resident.ranges.size() == 1); + EXPECT_TRUE(vmas_resident.ranges[0].offset == 0); + EXPECT_TRUE((uint64_t)vmas_resident.ranges[0].length == resident_total_size); + + close(test_file_fd); + remove(test_file.c_str()); +} + +TEST(meminspect_test, custom_probe_coverage_matches_with_probe) { + ZipMemInspector inspector(""); + VmaRangeGroup* probe = new VmaRangeGroup(); + probe->ranges.push_back(VmaRange(0, 500)); + probe->ranges.push_back(VmaRange(700, 100)); + probe->ranges.push_back(VmaRange(1000, 500)); + probe->ranges.push_back(VmaRange(2000, 100)); + // Probed Resident Memory Offset ranges: + // [0,500],[700,800],[1000,1500],[2000,2100] + EXPECT_EQ(probe->compute_total_size(), (unsigned long long)1200); + inspector.set_existing_probe(probe); + + // Emulate reading some files from the zip to compute their coverages + // fake1 memory offset ranges [100,300] + ZipEntryInfo info; + info.name = "fake1"; + info.offset_in_zip = 100; + info.file_size_bytes = 200; + inspector.add_file_info(info); + + // fake2 memory offset ranges [600,1200] + ZipEntryInfo info2; + info2.name = "fake2"; + info2.offset_in_zip = 600; + info2.file_size_bytes = 600; + inspector.add_file_info(info2); + + inspector.compute_per_file_coverage(); + std::vector<ZipEntryCoverage> coverages = inspector.get_file_coverages(); + EXPECT_EQ(coverages.size(), (size_t)2); + + // Result coverage for fake1 should be: [100,300] + EXPECT_EQ(coverages[0].coverage.ranges[0].offset, (uint32_t)100); + EXPECT_EQ(coverages[0].coverage.ranges[0].length, (uint32_t)200); + EXPECT_EQ(coverages[0].coverage.compute_total_size(), (unsigned long long)200); + EXPECT_EQ(coverages[0].info.name, "fake1"); + EXPECT_EQ(coverages[0].info.offset_in_zip, (uint32_t)100); + EXPECT_EQ(coverages[0].info.file_size_bytes, (uint32_t)200); + + // coverage coverage for fake2 should be: [700,800] and [1000,1200] + EXPECT_EQ(coverages[1].coverage.ranges[0].offset, (uint32_t)700); + EXPECT_EQ(coverages[1].coverage.ranges[0].length, (uint32_t)100); + EXPECT_EQ(coverages[1].coverage.ranges[1].offset, (uint32_t)1000); + EXPECT_EQ(coverages[1].coverage.ranges[1].length, (uint32_t)200); + EXPECT_EQ(coverages[1].coverage.compute_total_size(), (unsigned long long)300); // 100 + + // 200 + EXPECT_EQ(coverages[1].info.name, "fake2"); + EXPECT_EQ(coverages[1].info.offset_in_zip, (uint32_t)600); + EXPECT_EQ(coverages[1].info.file_size_bytes, (uint32_t)600); +} + +TEST(meminspect_test, whole_file_coverage_against_probe) { + ZipMemInspector inspector(""); + + // Emulate reading some files from the zip to compute their coverages + // fake1 memory offset ranges [100,300] + ZipEntryInfo info; + info.name = "fake1"; + info.offset_in_zip = 100; + info.file_size_bytes = 200; + inspector.add_file_info(info); + + // fake2 memory offset ranges [600,1200] + ZipEntryInfo info2; + info2.name = "fake2"; + info2.offset_in_zip = 600; + info2.file_size_bytes = 600; + inspector.add_file_info(info2); + + inspector.compute_per_file_coverage(); + std::vector<ZipEntryCoverage> coverages = inspector.get_file_coverages(); + EXPECT_EQ(coverages.size(), (size_t)2); + + // Check that coverage matches entire file sizes + EXPECT_EQ(coverages[0].coverage.ranges[0].offset, (uint32_t)100); + EXPECT_EQ(coverages[0].coverage.ranges[0].length, (uint32_t)200); + EXPECT_EQ(coverages[0].coverage.compute_total_size(), (unsigned long long)200); + EXPECT_EQ(coverages[0].info.name, "fake1"); + EXPECT_EQ(coverages[0].info.offset_in_zip, (uint32_t)100); + EXPECT_EQ(coverages[0].info.file_size_bytes, (uint32_t)200); + + EXPECT_EQ(coverages[1].coverage.ranges[0].offset, (uint32_t)600); + EXPECT_EQ(coverages[1].coverage.ranges[0].length, (uint32_t)600); + EXPECT_EQ(coverages[1].coverage.compute_total_size(), (unsigned long long)600); + EXPECT_EQ(coverages[1].info.name, "fake2"); + EXPECT_EQ(coverages[1].info.offset_in_zip, (uint32_t)600); + EXPECT_EQ(coverages[1].info.file_size_bytes, (uint32_t)600); +} + +TEST(meminspect_test, file_multiple_ranges_matches_probe) { + VmaRangeGroup probe; + probe.ranges.push_back(VmaRange(0, 500)); + probe.ranges.push_back(VmaRange(700, 100)); + probe.ranges.push_back(VmaRange(1000, 500)); + probe.ranges.push_back(VmaRange(2000, 100)); + // Probed Resident Memory Offset ranges: + // [0,500],[700,800],[1000,1500],[2000,2100] + EXPECT_EQ(probe.compute_total_size(), (unsigned long long)1200); + + std::vector<ZipEntryCoverage> desired_coverages; + + // fake1 file resides between [100,1100] + // desired ranges are [100,200],[400,710],[820,850] + ZipEntryCoverage file1_mem; + file1_mem.info.name = "fake1"; + file1_mem.info.offset_in_zip = 100; + file1_mem.info.file_size_bytes = 1000; + file1_mem.coverage.ranges.push_back(VmaRange(100, 100)); + file1_mem.coverage.ranges.push_back(VmaRange(400, 310)); + file1_mem.coverage.ranges.push_back(VmaRange(820, 30)); + desired_coverages.push_back(file1_mem); + + // fake2 memory offset ranges [1300,2100] + // desired ranges are [1400,1500],[1600,1650],[1800,2050] + ZipEntryCoverage file2_mem; + file2_mem.info.name = "fake2"; + file2_mem.info.offset_in_zip = 1300; + file2_mem.info.file_size_bytes = 750; + file2_mem.coverage.ranges.push_back(VmaRange(1400, 100)); + file2_mem.coverage.ranges.push_back(VmaRange(1600, 50)); + file2_mem.coverage.ranges.push_back(VmaRange(1800, 250)); + desired_coverages.push_back(file2_mem); + + std::vector<ZipEntryCoverage> coverages = + ZipMemInspector::compute_coverage(desired_coverages, &probe); + + EXPECT_EQ(coverages.size(), (size_t)2); + + // Result coverage for fake1 should be: [100,200],[400,500],[700,710] + EXPECT_EQ(coverages[0].coverage.ranges[0].offset, (uint32_t)100); + EXPECT_EQ(coverages[0].coverage.ranges[0].length, (uint32_t)100); + EXPECT_EQ(coverages[0].coverage.ranges[1].offset, (uint32_t)400); + EXPECT_EQ(coverages[0].coverage.ranges[1].length, (uint32_t)100); + EXPECT_EQ(coverages[0].coverage.ranges[2].offset, (uint32_t)700); + EXPECT_EQ(coverages[0].coverage.ranges[2].length, (uint32_t)10); + + EXPECT_EQ(coverages[0].coverage.compute_total_size(), (unsigned long long)210); + EXPECT_EQ(coverages[0].info.name, "fake1"); + EXPECT_EQ(coverages[0].info.offset_in_zip, (uint32_t)100); + EXPECT_EQ(coverages[0].info.file_size_bytes, (uint32_t)1000); + + // coverage coverage for fake2 should be: [1400,1500],[2000,2050] + EXPECT_EQ(coverages[1].coverage.ranges[0].offset, (uint32_t)1400); + EXPECT_EQ(coverages[1].coverage.ranges[0].length, (uint32_t)100); + EXPECT_EQ(coverages[1].coverage.ranges[1].offset, (uint32_t)2000); + EXPECT_EQ(coverages[1].coverage.ranges[1].length, (uint32_t)50); + EXPECT_EQ(coverages[1].coverage.compute_total_size(), (unsigned long long)150); + EXPECT_EQ(coverages[1].info.name, "fake2"); + EXPECT_EQ(coverages[1].info.offset_in_zip, (uint32_t)1300); + EXPECT_EQ(coverages[1].info.file_size_bytes, (uint32_t)750); +} + +TEST(meminspect_test, range_alignment_and_merge_matches) { + ZipMemInspector inspector(""); + VmaRangeGroup* probe = new VmaRangeGroup(); + probe->ranges.push_back(VmaRange(0, 500)); + probe->ranges.push_back(VmaRange(700, 100)); + int page_size = 4096; + + // Probed Resident Memory Offset ranges: + // [0,500],[700,800] + inspector.set_existing_probe(probe); + + // When we page align, we should end up with [0,500],[0,800] + align_ranges(probe->ranges, page_size); + EXPECT_EQ(probe->ranges[0].offset, (uint32_t)0); + EXPECT_EQ(probe->ranges[0].length, (uint32_t)500); + EXPECT_EQ(probe->ranges[1].offset, (uint32_t)0); + EXPECT_EQ(probe->ranges[1].length, (uint32_t)800); + EXPECT_EQ(probe->ranges.size(), (uint32_t)2); + + // Because we have overlapping ranges, a union-merge should + // skip duplication of intersections and end up with [0,800] + std::vector<VmaRange> merged = merge_ranges(probe->ranges); + EXPECT_EQ(merged[0].offset, (uint32_t)0); + EXPECT_EQ(merged[0].length, (uint32_t)800); + EXPECT_EQ(merged.size(), (uint32_t)1); +}
\ No newline at end of file diff --git a/pinner/tests/pintool_tests.cpp b/pinner/tests/pintool_tests.cpp new file mode 100644 index 00000000..f351a90a --- /dev/null +++ b/pinner/tests/pintool_tests.cpp @@ -0,0 +1,137 @@ +#include <gtest/gtest.h> +#include <stdio.h> +#include <list> + +#include <pin_utils.h> + +using namespace std; + +TEST(pintool_test, pinlist_matches_memranges) { + vector<VmaRange> vma_ranges; + unsigned int page_size = sysconf(_SC_PAGESIZE); + vma_ranges.push_back(VmaRange(0, 500)); + vma_ranges.push_back(VmaRange(5000, 5500)); + vma_ranges.push_back(VmaRange(21000, 13000)); + vma_ranges.push_back(VmaRange(50000, 35000)); + + string test_file = "/data/local/tmp/pintool_test"; + write_pinlist_file(test_file, vma_ranges); + + vector<VmaRange> read_ranges; + read_pinlist_file(test_file, read_ranges); + + EXPECT_EQ(vma_ranges.size(), read_ranges.size()); + for (size_t i = 0; i < vma_ranges.size(); ++i) { + // We expect to write pinlists that are page-aligned, so + // we compare against page aligned offsets. + uint64_t unaligned_bytes = vma_ranges[i].offset % page_size; + EXPECT_EQ(vma_ranges[i].offset - unaligned_bytes, read_ranges[i].offset); + EXPECT_EQ(vma_ranges[i].length + unaligned_bytes, read_ranges[i].length); + } + + remove(test_file.c_str()); +} + +TEST(pintool_test, pinlist_quota_applied) { + vector<VmaRange> vma_ranges; + unsigned int page_size = sysconf(_SC_PAGESIZE); + vma_ranges.push_back(VmaRange(0, 100)); + vma_ranges.push_back(VmaRange(page_size, 500)); + vma_ranges.push_back(VmaRange(page_size * 2, 300)); + vma_ranges.push_back(VmaRange(page_size * 3, 200)); + + const int ranges_to_write = 700; + string test_file = "/data/local/tmp/pintool_test"; + write_pinlist_file(test_file, vma_ranges, ranges_to_write); + + vector<VmaRange> read_ranges; + read_pinlist_file(test_file, read_ranges); + + int total_length = 0; + for (size_t i = 0; i < read_ranges.size(); ++i) { + total_length += read_ranges[i].length; + } + EXPECT_EQ(total_length, ranges_to_write); + + remove(test_file.c_str()); +} + +TEST(pintool_test, pinconfig_filter_coverage_matches) { + VmaRangeGroup* probe = new VmaRangeGroup(); + probe->ranges.push_back(VmaRange(0, 500)); + probe->ranges.push_back(VmaRange(1000, 5000)); + + ZipMemInspector* inspector = new ZipMemInspector(""); + + // Probed Resident Memory Offset ranges: + // [0,500],[1000,6000] + probe->compute_total_size(); + inspector->set_existing_probe(probe); + + // Emulate reading some files from the zip to compute their coverages + // fake1 memory offset ranges [100,400] + ZipEntryInfo info; + info.name = "fake1"; + info.offset_in_zip = 100; + info.file_size_bytes = 300; + inspector->add_file_info(info); + + // fake2 memory offset ranges [600,3000] + ZipEntryInfo info2; + info2.name = "fake2"; + info2.offset_in_zip = 600; + info2.file_size_bytes = 2400; + inspector->add_file_info(info2); + + ZipEntryInfo info3; + info2.name = "fake3"; + info2.offset_in_zip = 3100; + info2.file_size_bytes = 200; + inspector->add_file_info(info3); + + // Create a fake pinconfig file + PinConfig* pinconfig = new PinConfig(); + + // First file we want it entirely so don't provide ranges + PinConfigFile pinconfig_file_1; + pinconfig_file_1.filename = "fake1"; + pinconfig->files_.push_back(pinconfig_file_1); + + // Add a partially matched file + PinConfigFile pinconfig_file_2; + pinconfig_file_2.filename = "fake2"; + pinconfig_file_2.ranges.push_back(VmaRange(100, 500)); + pinconfig_file_2.ranges.push_back(VmaRange(800, 200)); + pinconfig->files_.push_back(pinconfig_file_2); + + // Add a file that does not exist + PinConfigFile pinconfig_file_3; + pinconfig_file_3.filename = "fake4"; + pinconfig_file_3.ranges.push_back(VmaRange(0, 1000)); + pinconfig->files_.push_back(pinconfig_file_3); + + PinTool pintool(""); + pintool.set_custom_zip_inspector(inspector); + pintool.compute_zip_entry_coverages(); + pintool.filter_zip_entry_coverages(pinconfig); + + std::vector<ZipEntryCoverage> filtered = pintool.get_filtered_zip_entries(); + + // We only matched 2 files, one should not have matched to any filter. + EXPECT_EQ(filtered.size(), (unsigned long)2); + EXPECT_EQ(filtered[0].info.name, "fake1"); + EXPECT_EQ(filtered[0].coverage.ranges[0].offset, (unsigned long)100); + EXPECT_EQ(filtered[0].coverage.ranges[0].length, (unsigned long)300); + + // Probe Resident has [0,500],[1000,6000]. + // fake2 file lives within [600,3000] + // fake2 relative offsets from pinconfig [100,600],[800,1000] + // fake2 absolute zip offsets are [700,1200],[1400,1600] + // then matching absolute offsets against resident yields [1000,1200],[1400,1600] + EXPECT_EQ(filtered[1].info.name, "fake2"); + EXPECT_EQ(filtered[1].info.offset_in_zip, (unsigned long)600); + EXPECT_EQ(filtered[1].coverage.ranges[0].offset, (unsigned long)1000); + EXPECT_EQ(filtered[1].coverage.ranges[0].length, (unsigned long)200); + EXPECT_EQ(filtered[1].coverage.ranges[1].offset, (unsigned long)1400); + EXPECT_EQ(filtered[1].coverage.ranges[1].length, (unsigned long)200); +}
\ No newline at end of file diff --git a/profcollectd/libprofcollectd/config.rs b/profcollectd/libprofcollectd/config.rs index 14236ab9..87242489 100644 --- a/profcollectd/libprofcollectd/config.rs +++ b/profcollectd/libprofcollectd/config.rs @@ -30,7 +30,8 @@ use std::time::Duration; const PROFCOLLECT_CONFIG_NAMESPACE: &str = "profcollect_native_boot"; const PROFCOLLECT_NODE_ID_PROPERTY: &str = "persist.profcollectd.node_id"; -const DEFAULT_BINARY_FILTER: &str = "^/(system|apex/.+)/(bin|lib|lib64)/.+"; +const DEFAULT_BINARY_FILTER: &str = + "(^/(system|apex/.+|vendor)/(bin|lib|lib64)/.+)|kernel.kallsyms"; pub const REPORT_RETENTION_SECS: u64 = 14 * 24 * 60 * 60; // 14 days. // Static configs that cannot be changed. |