diff options
author | Edgar Arriaga <edgararriaga@google.com> | 2023-09-12 18:47:31 +0000 |
---|---|---|
committer | Edgar Arriaga <edgararriaga@google.com> | 2023-10-23 18:47:45 +0000 |
commit | 28392b53237a6615b2b9609a534b8dce37609a5c (patch) | |
tree | 18a6a417b99eeaab24413892d48964515d5e1f64 /pinner | |
parent | 62816c9f4c8f5cb116af34d1c26455e351e6d3d6 (diff) | |
download | extras-28392b53237a6615b2b9609a534b8dce37609a5c.tar.gz |
PinTool add support contextual information and filtering
Adding support for inspecting zip entries within a zip file
and providing per-file memory information for them. Also added
support for filtering based on a pinconfig file to further constrain
probes or to only generate pinlist.meta files for a certain set of
files within a zip file.
Test: meminspect_tests
Test: pintool_tests
Bug: 297095632
Change-Id: Ie1a75f44606c4cd7a0971a09322169c01bdca325
Diffstat (limited to 'pinner')
-rw-r--r-- | pinner/Android.bp | 3 | ||||
-rw-r--r-- | pinner/README.md | 129 | ||||
-rw-r--r-- | pinner/include/meminspect.h | 234 | ||||
-rw-r--r-- | pinner/include/pin_utils.h | 96 | ||||
-rw-r--r-- | pinner/meminspect.cpp | 282 | ||||
-rw-r--r-- | pinner/pin_utils.cpp | 332 | ||||
-rw-r--r-- | pinner/pintool.cpp | 332 | ||||
-rw-r--r-- | pinner/sample_pinconfig.txt | 5 | ||||
-rw-r--r-- | pinner/tests/Android.bp | 19 | ||||
-rw-r--r-- | pinner/tests/TEST_MAPPING | 10 | ||||
-rw-r--r-- | pinner/tests/meminspect_tests.cpp | 195 | ||||
-rw-r--r-- | pinner/tests/pintool_tests.cpp | 119 |
12 files changed, 1573 insertions, 183 deletions
diff --git a/pinner/Android.bp b/pinner/Android.bp index df922e0b..40197ba2 100644 --- a/pinner/Android.bp +++ b/pinner/Android.bp @@ -26,7 +26,9 @@ cc_library_static { ], shared_libs: [ "libbase", + "libziparchive", ], + export_shared_lib_headers: ["libziparchive"], export_include_dirs: ["include"], cppflags: [ "-g", @@ -40,6 +42,7 @@ cc_binary { srcs: ["pintool.cpp"], shared_libs: [ "libbase", + "libziparchive", ], static_libs: [ "libmeminspect", diff --git a/pinner/README.md b/pinner/README.md index f7b37f88..2115e809 100644 --- a/pinner/README.md +++ b/pinner/README.md @@ -1,6 +1,13 @@ # Pin Tool -This tool is used for multiple operations related to memory pinning and inspection. +This tool is currently used mainly for: + +1) Inspecting resident memory and locality +2) Generating and inspecting pinlist.meta used by Android's PinnerService + +For memory inspection, it allows probing live memory and providing the memory +locations that are resident which can be used for diagnosis or as part of PGO +optimizations. ## Build and Install the tool @@ -12,33 +19,119 @@ adb shell cd /data/local/tmp/pintool ``` + ## How to use the tool to generate pinner files -One of the core usages of the tool is to inspect resident memory and -generate pinlist.meta files which are then consumed by the Android PinnerService -to pin memory (mlock) and avoid it from being evicted. +Here are some sample use cases for this tool. + +### Sample Use Case 1: Probe Resident Memory for a mapped file or library and dump to console + +Executing the following command and providing the path to a library mapped by a process +it will dump the resident memory ranges. + +``` +./pintool file <path_to_your_library> --gen-probe --dump +``` + +Note: you can use any kind of mapped file such as .so, .apk, etc. + +### Sample Use Case 2: Probe per-file Resident Memory for a mapped apk file and dump to console + +Executing this command will inspect resident memory for a zip file and dump +per-file breakdowns. + +``` +./pintool file <path_to_myfile.apk> --zip --gen-probe --dump +``` + +### Sample Use Case 3: Probe per-file resident memory for a mapped apk, dump and generate pinlist.meta +``` +./pintool file <path_to_myfile.apk> --zip --gen-probe --dump -o pinlist.meta +``` + +### Sample Use Case 4: Probe per-file resident memory and filter it with a provided pinconfig +``` +./pintool file <path_to_myfile.apk> --zip --gen-probe --pinconfig pinconfig.txt --dump -o pinlist.meta +``` + +### Sample Use Case 5: Dump contents of a provided pinlist.meta file +``` +./pintool pinlist pinlist.meta --dump -v +``` + +### Sample Use Case 6: Read a zip and filter based on a pinconfig file and generate pinlist.meta without probing + +This will skip doing any probing and it will just apply filtering based on the pinconfig.txt, this is helpful +in cases where you do not intend to do any kind of PGO probing and know exactly what ranges you want to pin within your file + +``` +./pintool file <path_to_myfile.apk> --zip --pinconfig pinconfig.txt --dump -o pinlist.meta +``` + +### Sample Use Case 7: Load an existing zip probe and inspect its per-file contents + +``` +./pintool file /data/app/~~tmTrs5_XINwbpYWroRu5rA==/org.chromium.trichromelibrary_602300034-EFoOwMgVNBbwkMnp9zcWbg==/base.apk --zip --use-probe pinlist.meta --dump +``` -A sample usage of this tool in a PGO style fashion can be like this: +## Pinconfig File Structure -This is a sample flow of how to use this tool: -1. Run the app that makes use of the file you want to pin. To have resident -memory for the file. +Pinconfig files specify a custom filter to be applied on top of a generated or provided memory probe +it should specify a subset of files and optionally ranges within those files to be matched against +and subsequently kept in case a pinlist.meta is generated. -2. Run this program to generate the pinlist.meta file. +A `pinconfig.txt` is just a list of files with a key value pair separated by a newline. + +`pinconfig.txt` structure pattern: ``` -./pintool probe -p <file_to_probe> <options> +(file <file> +[offset <value> +length <value>])* ``` -This will inspect the memory for the provided <file_to_probe> and generate -a pinlist.meta file with the resident memory ranges. +where: +<file> + Filename as a string, the parser will do a contains like operation (e.g. GLOB(*<file>*)) to match against files + within the zip file and stop on first match. +<value> + Unsigned integer value + +Note: `offset` and `length` tokens are optional and if ommited, the whole file will be considered desired. + -Note: Running ./pintool to obtain more usage documentation. +Example `pinconfig.txt`: +``` +file lib/arm64-v8a/libcrashpad_handler_trampoline.so +file libmonochrome_64.so +offset 1000 +len 50000 +file libarcore_sdk_c.so +``` + +## Pinlist.meta files + +"pinlist.meta" files are consumed by PinnerService. + +These files specify a list of memory ranges to be pinned (mlocked). +If Android's PinnerService allows your app pinning, it will read the pinlist.meta +file from inside your apk's assets folder (assets/pinlist.meta) and pin based +on the specified ranges. + +Note: The PinnerService will need to support pinning your apk in order for the +pinlist.meta file to be used. + +A pinlist.meta file is a binary file with a set of tuples of OFFSET and LENGTH +stored in Big Endian format. + +4 byte: OFFSET +4 byte: LEN + +pinlist.meta +``` +OFFSET LEN* +``` -3. Output "pinlist.meta" can be incorporate it within your build -process to have it bundled inside your apk (inside assets/<file_to_pin.apk>) -to have the PinnerService know what memory ranges to pin within your apk. -Note: The PinnerService will need to support pinning your apk, so you may -need to explicitly request a pin. +So to read those files, it is usually helpful to use the `pintool`. ## Other potential uses diff --git a/pinner/include/meminspect.h b/pinner/include/meminspect.h index 11cca2ba..a2e09ec2 100644 --- a/pinner/include/meminspect.h +++ b/pinner/include/meminspect.h @@ -10,6 +10,7 @@ #include <iostream> #include <string> #include <vector> +#include "ziparchive/zip_archive.h" #define MEMINSPECT_FAIL_OPEN 1 #define MEMINSPECT_FAIL_FSTAT 2 @@ -26,25 +27,242 @@ class VmaRange { uint32_t offset; uint32_t length; + VmaRange() {} VmaRange(uint32_t off, uint32_t len) : offset(off), length(len) {} + + bool is_empty() const; + + /** + * @brief Compute the intersection of this range with another range + * + * Intersection Operation: + * + * Example 1: + * [ Range A ] + * [ Range B ] + * Intersection: + * [ C ] + * + * Example 2: + * [ Range A ] [ Range B ] + * No Intersection + * + * @param target range to test against + * @return the intersection range, if none is found, empty range is returned. + */ + VmaRange intersect(const VmaRange& target) const; + + /** + * @brief Merges the current range with a target range using a union operation + * that is only successful when overlapping ranges occur. + * A visual explanation can be seen as: + * + * Union-merge Operation: + * + * Example 1: + * [ Range A ] + * [ Range B ] + * Merged: + * [ Range C ] + * + * Example 2: + * [ Range A ] [ Range B ] + * Fails, no merge available. + * + * @param target The range to test against. + * @param result Upon successfully merging, contains the resulting range. + * @return the merged range, if none is found, empty range is returned. + */ + VmaRange union_merge(const VmaRange& target) const; + + uint32_t end_offset() const; }; -struct ResidentMemResult { - public: - std::vector<VmaRange> resident_memory_ranges; +/** + * Represents a set of memory ranges + */ +struct VmaRangeGroup { + std::vector<VmaRange> ranges; + + /** + * Compute intersection coverage between |range| and |this->ranges| + * and append it to |out_memres| + */ + void compute_coverage(const VmaRange& range, VmaRangeGroup& out_memres) const; + + /** + * Apply an offset to all existing |ranges|. + */ + void apply_offset(uint64_t offset); + + /** + * Computes total resident bytes from existing set of memory ranges. + */ + uint64_t compute_total_size(); +}; + +/** + * Represents useful immutable metadata for zip entry + */ +struct ZipEntryInfo { + std::string name; + uint64_t offset_in_zip; uint64_t file_size_bytes; - uint64_t total_resident_bytes; + uint64_t uncompressed_size; }; /** + * Represents the resident memory coverage for a zip entry within a zip file. + */ +struct ZipEntryCoverage { + ZipEntryInfo info; + + /** + * Contains all the coverage ranges if any have been computed with |compute_coverage| + * and their offsets will be the absolute global offset from the zip file start. + */ + VmaRangeGroup coverage; + + /** + * Computes the intersection coverage for the current zip file entry + * resident memory against a provided |probe| representing another set + * of ranges. + */ + ZipEntryCoverage compute_coverage(const VmaRangeGroup& probe) const; +}; + +// Class used for inspecting resident memory for entries within a zip file +class ZipMemInspector { + /** + * Stored probe of resident ranges either computed or provided by user. + */ + VmaRangeGroup* probe_resident_ = nullptr; + + /** + * List of file entries within zip file. + */ + std::vector<ZipEntryInfo> entry_infos_; + + /** + * Path to zip file. + */ + std::string filename_; + + /** + * Result of computing coverage operations. + */ + std::vector<ZipEntryCoverage> entry_coverages_; + + /** + * Handle that allows reading the zip entries. + */ + ZipArchiveHandle handle_; + + public: + ZipMemInspector(std::string filename) : filename_(filename) {} + ~ZipMemInspector(); + + /** + * Reads zip file and computes resident memory coverage per zip entry if + * a probe is provided, if no probe is provided, then whole file coverage + * will be assumed. + * + * Note: If any zip entries have been manually added via |add_file_info| + * then coverage will be only computed against manually added entries. + * + * @return 0 on success and 1 on error + */ + int compute_per_file_coverage(); + + /** + * Computes resident memory for the entire zip file. + * + * @return 0 on success, 1 on failure + */ + int probe_resident(); + + /** + * Retrieves the currently set probe if any exists. + */ + VmaRangeGroup* get_probe(); + + /** + * Sets probe data in case you decide to pass a previously taken probe instead of a live taken + * one. + */ + void set_existing_probe(VmaRangeGroup* probe); + + /** + * Returns the result of memory coverage of each file if any has been computed via + * |compute_per_file_coverage|. + */ + std::vector<ZipEntryCoverage>& get_file_coverages(); + + /** + * Returns the file information for each zip entry. + */ + std::vector<ZipEntryInfo>& get_file_infos(); + + /** + * Add a zip entry manually. + * + * Note: Zip entries are usually retrieved by reading the |filename_| so + * this method is mostly used for cases where client wants control of + * zip file reading or for testing. + */ + void add_file_info(ZipEntryInfo& file); + + /** + * Computes the intersection coverage between provided |files| and |probe|. + * + * @return result of coverage computation + */ + static std::vector<ZipEntryCoverage> compute_coverage( + const std::vector<ZipEntryCoverage>& files, VmaRangeGroup* probe); + + private: + /** + * Read files and zip relative offsets for them. + * + * @return 0 on success, 1 on failure. + */ + int read_files_and_offsets(); +}; + +/** + * Retrieve file size in bytes for |file| + * + * @return positive value with file size on success, otherwise, returns -1 on error. + */ +int64_t get_file_size(const std::string& file); + +/** * @brief Probe resident memory for a currently opened file in the system. * * @param probed_file File to probe as defined by its path. * @param out_resident_mem Inspection result. This is populated when called. - * @param pages_per_mincore Size of mincore window used, bigger means more memory but slightly - * faster. + * @param pages_per_mincore Size of mincore window used, bigger means more memory used + * during operation but slightly faster. * @return 0 on success or on failure a non-zero error code from the following list: * MEMINSPECT_FAIL_OPEN, MEMINSPECT_FAIL_FSTAT, MEMINSPECT_FAIL_MINCORE */ -int probe_resident_memory(std::string probed_file, ResidentMemResult& out_resident_mem, - int pages_per_mincore = DEFAULT_PAGES_PER_MINCORE);
\ No newline at end of file +int probe_resident_memory(std::string probed_file, VmaRangeGroup& out_resident_mem, + int pages_per_mincore = DEFAULT_PAGES_PER_MINCORE); + +/** + * @brief Align vma ranges to a certain page size + * + * @param ranges vma ranges that have to be aligned + * @param alignment Desired alignment, this is usually the page size. + */ +void align_ranges(std::vector<VmaRange>& ranges, unsigned int alignment); + +/** + * @brief Merges a list of ranges following a union-like merge which + * means that two ranges that overlap will avoid double accounting for + * overlaps. + * + * @param ranges vma ranges that need to be merged. + * @return new vector with ranges merged. + */ +std::vector<VmaRange> merge_ranges(const std::vector<VmaRange>& ranges);
\ No newline at end of file diff --git a/pinner/include/pin_utils.h b/pinner/include/pin_utils.h index 8db0e8aa..b6066d63 100644 --- a/pinner/include/pin_utils.h +++ b/pinner/include/pin_utils.h @@ -1,7 +1,23 @@ #pragma once +#include <list> #include "meminspect.h" +struct PinConfigFile { + std::string filename; + + // File relative offsets + std::vector<VmaRange> ranges; + + ZipEntryCoverage to_zipfilemem(const ZipEntryInfo& info); +}; + +struct PinConfig { + std::list<PinConfigFile> files_; + + int parse(std::string filename, bool verbose = false); +}; + /** * @brief Generate a pinlist file from a given list of vmas containing a list of 4-byte pairs * representing (4-byte offset, 4-byte len) contiguous in memory and they are stored in big endian @@ -9,9 +25,12 @@ * * @param output_file Output file to write pinlist * @param vmas_to_pin Set of vmas to write into pinlist file. + * @param write_quota Specifies a maximum amount o bytes to be written to the pinlist file + * or -1 means no limit. * @return 0 on success, non-zero on failure */ -int write_pinlist_file(const std::string& output_file, const std::vector<VmaRange>& vmas_to_pin); +int write_pinlist_file(const std::string& output_file, const std::vector<VmaRange>& vmas_to_pin, + int64_t write_quota = -1); /** * @brief This method is the counter part of @see write_pinlist_file(). It will read an existing @@ -21,4 +40,77 @@ int write_pinlist_file(const std::string& output_file, const std::vector<VmaRang * @param pinranges Vmas read from pinlist file. This is populated on call. * @return 0 on success, non-zero on failure */ -int read_pinlist_file(const std::string& pinner_file, /*out*/ std::vector<VmaRange>& pinranges);
\ No newline at end of file +int read_pinlist_file(const std::string& pinner_file, /*out*/ std::vector<VmaRange>& pinranges); + +enum ProbeType { + UNSET, // No probe setup + GENERATE, // Generate a probe + CUSTOM // User generated probe +}; + +class PinTool { + public: + enum DumpType { PROBE, FILE_COVERAGE, FILTERED }; + + private: + std::string input_file_; + std::string custom_probe_file_; + PinConfig* pinconfig_; + std::vector<ZipEntryCoverage> filtered_files_; + bool verbose_; + ZipMemInspector* zip_inspector_ = nullptr; + + public: + PinTool(const std::string& input_file) : input_file_(input_file) { + zip_inspector_ = new ZipMemInspector(input_file_); + } + + ~PinTool() { + delete zip_inspector_; + delete pinconfig_; + } + + void set_verbose_output(bool verbose); + + // Read |probe_file| which should be a pinlist.meta style + // file and use it as current probe. + void read_probe_from_pinlist(std::string probe_file); + + // Compute a resident memory probe for |input_file_| + int probe_resident(); + + // Compute coverage for each zip entry contained within + // |input_file_|. + // Note: It only works for zip files + void compute_zip_entry_coverages(); + + /** + * Filter coverages based on a provided pinconfig style file + * See README.md for sample structure of pinconfig file. + * + * Note: It only works for zip files, for non zip files, this will be + * a no-op. + */ + void filter_zip_entry_coverages(const std::string& pinconfig_file); + + void filter_zip_entry_coverages(PinConfig* pinconfig); + + /** + * Dumps output of existing coverages to console for |type|. + */ + void dump_coverages(DumpType type); + + /** + * Writes coverages into a pinlist.meta style file. + * + * @param write_quota Maximum bytes allowed to be written to file. + */ + void write_coverages_as_pinlist(std::string output_pinlist, int64_t write_quota = -1); + + std::vector<ZipEntryCoverage> get_filtered_zip_entries(); + + /** + * Sets a user defined inspector, currently only used for testing. + */ + void set_custom_zip_inspector(ZipMemInspector* inspector); +};
\ No newline at end of file diff --git a/pinner/meminspect.cpp b/pinner/meminspect.cpp index 181129ad..84e00928 100644 --- a/pinner/meminspect.cpp +++ b/pinner/meminspect.cpp @@ -1,45 +1,163 @@ #include "meminspect.h" #include <android-base/unique_fd.h> +#include "ziparchive/zip_archive.h" using namespace std; using namespace android::base; using namespace ::android::base; +const static VmaRange VMA_RANGE_EMPTY = VmaRange(0, 0); + +uint32_t VmaRange::end_offset() const { + return offset + length; +} + +uint64_t VmaRangeGroup::compute_total_size() { + uint64_t total_size = 0; + for (auto&& range : ranges) { + total_size += range.length; + } + return total_size; +} + +void VmaRangeGroup::apply_offset(uint64_t offset) { + for (auto&& range : ranges) { + range.offset += offset; + } +} + +void VmaRangeGroup::compute_coverage(const VmaRange& range, VmaRangeGroup& out_memres) const { + for (auto&& resident_range : ranges) { + VmaRange intersect_res = resident_range.intersect(range); + if (!intersect_res.is_empty()) { + out_memres.ranges.push_back(intersect_res); + } + } +} + +bool VmaRange::is_empty() const { + return length == 0; +} + +VmaRange VmaRange::intersect(const VmaRange& target) const { + // First check if the slice is outside our range + if (target.end_offset() <= this->offset) { + return VMA_RANGE_EMPTY; + } + if (target.offset >= this->end_offset()) { + return VMA_RANGE_EMPTY; + } + VmaRange result; + // the slice should now be inside the range so compute the intersection. + result.offset = std::max(target.offset, this->offset); + uint32_t res_end = std::min(target.end_offset(), end_offset()); + result.length = res_end - result.offset; + + return result; +} + +VmaRange VmaRange::union_merge(const VmaRange& target) const { + VmaRange result = intersect(target); + if (result.is_empty()) { + // Disjointed ranges, no merge. + return VMA_RANGE_EMPTY; + } + + // Since there is an intersection, merge ranges between lowest + // and highest value. + result.offset = std::min(offset, target.offset); + uint32_t res_end = std::max(target.end_offset(), end_offset()); + result.length = res_end - result.offset; + return result; +} + +void align_ranges(std::vector<VmaRange>& vmas_to_align, unsigned int alignment) { + for (auto&& vma_to_align : vmas_to_align) { + uint32_t unaligned_offset = vma_to_align.offset % alignment; + vma_to_align.offset -= unaligned_offset; + vma_to_align.length += unaligned_offset; + } +} + +bool compare_range(VmaRange& a, VmaRange& b) { + return a.offset < b.offset; +} + +std::vector<VmaRange> merge_ranges(const std::vector<VmaRange>& ranges) { + if (ranges.size() <= 1) { + // Not enough ranges to perform a merge. + return ranges; + } + + std::vector<VmaRange> to_merge_ranges = ranges; + std::vector<VmaRange> merged_ranges; + // Sort the ranges to make a slightly more efficient merging. + std::sort(to_merge_ranges.begin(), to_merge_ranges.end(), compare_range); + + // The first element will always start as-is, then start merging with subsequent elements. + merged_ranges.push_back(to_merge_ranges[0]); + for (int iMerged = 0, iTarget = 1; iTarget < to_merge_ranges.size(); ++iTarget) { + VmaRange merged = merged_ranges[iMerged].union_merge(to_merge_ranges[iTarget]); + if (!merged.is_empty()) { + // Merge was successful, swallow range. + merged_ranges[iMerged] = merged; + } else { + // Merge failed, add disjointed range. + merged_ranges.push_back(to_merge_ranges[iTarget]); + ++iMerged; + } + } + + return merged_ranges; +} + +int64_t get_file_size(const std::string& file) { + unique_fd file_ufd(open(file.c_str(), O_RDONLY)); + int fd = file_ufd.get(); + if (fd == -1) { + return -1; + } + + struct stat fstat_res; + int res = fstat(fd, &fstat_res); + if (res == -1) { + return -1; + } + + return fstat_res.st_size; +} + int probe_resident_memory(string probed_file, - /*out*/ ResidentMemResult& memresult, int pages_per_mincore) { + /*out*/ VmaRangeGroup& resident_ranges, int pages_per_mincore) { unique_fd probed_file_ufd(open(probed_file.c_str(), O_RDONLY)); int probe_fd = probed_file_ufd.get(); if (probe_fd == -1) { return MEMINSPECT_FAIL_OPEN; } - struct stat fstat_res; - int res = fstat(probe_fd, &fstat_res); - if (res == -1) { + int64_t total_bytes = get_file_size(probed_file); + if (total_bytes < 0) { return MEMINSPECT_FAIL_FSTAT; } - unsigned long total_bytes = fstat_res.st_size; - memresult.file_size_bytes = total_bytes; - - char* base_address = (char*)mmap(0, total_bytes, PROT_READ, MAP_SHARED, probe_fd, /*offset*/ 0); + char* base_address = + (char*)mmap(0, (uint64_t)total_bytes, PROT_READ, MAP_SHARED, probe_fd, /*offset*/ 0); // this determines how many pages to inspect per mincore syscall unsigned char* window = new unsigned char[pages_per_mincore]; unsigned int page_size = sysconf(_SC_PAGESIZE); unsigned long bytes_inspected = 0; - unsigned long pages_in_memory = 0; // total bytes in inspection window unsigned long window_bytes = page_size * pages_per_mincore; - char* current_window_address; + char* window_base; bool started_vma_range = false; uint32_t resident_vma_start_offset = 0; - for (current_window_address = base_address; bytes_inspected < total_bytes; - current_window_address += window_bytes, bytes_inspected += window_bytes) { - int res = mincore(current_window_address, window_bytes, window); + for (window_base = base_address; bytes_inspected < total_bytes; + window_base += window_bytes, bytes_inspected += window_bytes) { + int res = mincore(window_base, window_bytes, window); if (res != 0) { if (errno == ENOMEM) { // Did not find page, maybe it's a hole. @@ -51,15 +169,13 @@ int probe_resident_memory(string probed_file, // and as soon as a change in residency happens a range is // created or finished. for (int iWin = 0; iWin < pages_per_mincore; ++iWin) { - if ((window[iWin] & 1) > 0) { + if ((window[iWin] & (unsigned char)1) != 0) { // Page is resident - ++pages_in_memory; if (!started_vma_range) { // End of range started_vma_range = true; uint32_t window_offset = iWin * page_size; - resident_vma_start_offset = - current_window_address + window_offset - base_address; + resident_vma_start_offset = window_base + window_offset - base_address; } } else { // Page is not resident @@ -67,11 +183,10 @@ int probe_resident_memory(string probed_file, // Start of range started_vma_range = false; uint32_t window_offset = iWin * page_size; - uint32_t resident_vma_end_offset = - current_window_address + window_offset - base_address; + uint32_t resident_vma_end_offset = window_base + window_offset - base_address; uint32_t resident_len = resident_vma_end_offset - resident_vma_start_offset; VmaRange vma_range(resident_vma_start_offset, resident_len); - memresult.resident_memory_ranges.push_back(vma_range); + resident_ranges.ranges.push_back(vma_range); } } } @@ -79,12 +194,131 @@ int probe_resident_memory(string probed_file, // This was the last window, so close any opened vma range if (started_vma_range) { started_vma_range = false; - uint32_t in_memory_vma_end = current_window_address - base_address; + uint32_t in_memory_vma_end = window_base - base_address; uint32_t resident_len = in_memory_vma_end - resident_vma_start_offset; VmaRange vma_range(resident_vma_start_offset, resident_len); - memresult.resident_memory_ranges.push_back(vma_range); + resident_ranges.ranges.push_back(vma_range); + } + + return 0; +} + +ZipMemInspector::~ZipMemInspector() { + CloseArchive(handle_); + delete probe_resident_; +} + +ZipEntryCoverage ZipEntryCoverage::compute_coverage(const VmaRangeGroup& probe) const { + ZipEntryCoverage file_coverage; + file_coverage.info = info; + + // Compute coverage for each range in file against probe which represents a set of ranges. + for (auto&& range : coverage.ranges) { + probe.compute_coverage(range, file_coverage.coverage); + } + + return file_coverage; +} + +std::vector<ZipEntryCoverage> ZipMemInspector::compute_coverage( + const std::vector<ZipEntryCoverage>& files, VmaRangeGroup* probe) { + if (probe == nullptr) { + // No probe to calculate coverage against, so coverage is zero. + return std::vector<ZipEntryCoverage>(); + } + + std::vector<ZipEntryCoverage> file_coverages; + // Find the file coverage against provided probe. + for (auto&& file : files) { + // For each file, compute coverage against the probe which represents a list of ranges. + ZipEntryCoverage file_coverage = file.compute_coverage(*probe); + file_coverages.push_back(file_coverage); } - memresult.total_resident_bytes = pages_in_memory * page_size; + return file_coverages; +} + +void ZipMemInspector::add_file_info(ZipEntryInfo& file) { + entry_infos_.push_back(file); +} + +int ZipMemInspector::compute_per_file_coverage() { + if (entry_infos_.empty()) { + // We haven't read the file information yet, so do it now. + if (read_files_and_offsets()) { + cerr << "Could not read zip entries to compute coverages." << endl; + return 1; + } + } + + // All existing files should consider their whole memory as present by default. + std::vector<ZipEntryCoverage> entry_coverages; + for (auto&& entry_info : entry_infos_) { + ZipEntryCoverage entry_coverage; + entry_coverage.info = entry_info; + VmaRange file_vma_range(entry_info.offset_in_zip, entry_info.file_size_bytes); + entry_coverage.coverage.ranges.push_back(file_vma_range); + entry_coverage.coverage.compute_total_size(); + entry_coverages.push_back(entry_coverage); + } + + if (probe_resident_ != nullptr) { + // We decided to compute coverage based on a probe + entry_coverages_ = compute_coverage(entry_coverages, probe_resident_); + } else { + // No probe means whole file coverage + entry_coverages_ = entry_coverages; + } + + return 0; +} + +VmaRangeGroup* ZipMemInspector::get_probe() { + return probe_resident_; +} + +void ZipMemInspector::set_existing_probe(VmaRangeGroup* probe) { + this->probe_resident_ = probe; +} + +std::vector<ZipEntryCoverage>& ZipMemInspector::get_file_coverages() { + return entry_coverages_; +} + +int ZipMemInspector::probe_resident() { + probe_resident_ = new VmaRangeGroup(); + int res = probe_resident_memory(filename_, *probe_resident_); + if (res != 0) { + // Failed to probe + return res; + } + + return 0; +} + +std::vector<ZipEntryInfo>& ZipMemInspector::get_file_infos() { + return entry_infos_; +} + +int ZipMemInspector::read_files_and_offsets() { + if (OpenArchive(filename_.c_str(), &handle_) < 0) { + return 1; + } + void* cookie; + int res = StartIteration(handle_, &cookie); + if (res != 0) { + return 1; + } + + ZipEntry64 entry; + string name; + while (Next(cookie, &entry, &name) == 0) { + ZipEntryInfo file; + file.name = name; + file.offset_in_zip = entry.offset; + file.file_size_bytes = entry.compressed_length; + file.uncompressed_size = entry.uncompressed_length; + entry_infos_.push_back(file); + } return 0; -}
\ No newline at end of file +} diff --git a/pinner/pin_utils.cpp b/pinner/pin_utils.cpp index 4469d781..7c746bb0 100644 --- a/pinner/pin_utils.cpp +++ b/pinner/pin_utils.cpp @@ -1,51 +1,347 @@ #include "pin_utils.h" +#include <android-base/parseint.h> +#include <algorithm> +#include <fstream> +#include <map> +#include <string_view> +#include <utility> +#include <vector> using namespace std; using namespace android::base; -int write_pinlist_file(const std::string& output_file, const std::vector<VmaRange>& vmas_to_pin) { - int pinlist_fd = open(output_file.c_str(), O_RDWR | O_CREAT, "w"); - if (pinlist_fd == -1) { +int write_pinlist_file(const std::string& output_file, + const std::vector<ZipEntryCoverage>& files_to_write, int64_t write_quota) { + std::vector<VmaRange> ranges; + for (auto&& file : files_to_write) { + ranges.insert(ranges.end(), file.coverage.ranges.begin(), file.coverage.ranges.end()); + } + return write_pinlist_file(output_file, ranges, write_quota); +} + +int write_pinlist_file(const std::string& output_file, const std::vector<VmaRange>& vmas_to_write, + int64_t write_quota) { + ofstream pinlist_file(output_file); + if (pinlist_file.fail()) { return 1; } - for (int i = 0; i < vmas_to_pin.size(); ++i) { - uint32_t vma_start_offset = vmas_to_pin[i].offset; - uint32_t vma_length = vmas_to_pin[i].length; + int64_t total_written = 0; + unsigned int page_size = sysconf(_SC_PAGESIZE); + const bool has_quota = write_quota > 0; + bool reached_quota = false; + + // The PinnerService does not require aligned offsets, however, aligning + // allows our summary results to be accurate and avoids over-accounting + // of pinning in PinnerService. + std::vector<VmaRange> processed_vmas_to_write = vmas_to_write; + align_ranges(processed_vmas_to_write, page_size); + + // When we page-align the ranges, we may cause overlaps between ranges + // as we elongate the begin offset to match the page the previous + // range may end up overlapping the current one. + processed_vmas_to_write = merge_ranges(processed_vmas_to_write); + + for (auto&& processed_vma_to_write : processed_vmas_to_write) { + uint32_t vma_start_offset = processed_vma_to_write.offset; + uint32_t vma_length = processed_vma_to_write.length; + if (has_quota && (total_written + vma_length > write_quota)) { + // We would go beyond quota, set the maximum allowed write and exit. + vma_length = write_quota - total_written; + reached_quota = true; + } // Transform to BigEndian as PinnerService requires that endianness for reading. uint32_t vma_start_offset_be = htobe32(vma_start_offset); uint32_t vma_length_be = htobe32(vma_length); - int write_res = write(pinlist_fd, &vma_start_offset_be, sizeof(vma_start_offset_be)); - if (write_res == -1) { + cout << "Pinlist Writing start=" << vma_start_offset << " bytes=" << vma_length << endl; + pinlist_file.write(reinterpret_cast<char*>(&vma_start_offset_be), + sizeof(vma_start_offset_be)); + if (pinlist_file.fail()) { return 1; } - write_res = write(pinlist_fd, &vma_length_be, sizeof(vma_length_be)); - if (write_res == -1) { + pinlist_file.write(reinterpret_cast<char*>(&vma_length_be), sizeof(vma_length_be)); + total_written += vma_length; + if (pinlist_file.fail()) { return 1; } + + if (reached_quota) { + break; + } } - close(pinlist_fd); return 0; } int read_pinlist_file(const std::string& pinner_file, /*out*/ std::vector<VmaRange>& pinranges) { - int pinlist_fd = open(pinner_file.c_str(), O_RDWR | O_CREAT, "w"); - if (pinlist_fd == -1) { + ifstream pinlist_file(pinner_file); + if (pinlist_file.fail()) { return 1; } uint32_t vma_start; uint32_t vma_length; - while (read(pinlist_fd, &vma_start, sizeof(vma_start)) > 0) { - int read_res = read(pinlist_fd, &vma_length, sizeof(vma_length)); - if (read_res == -1) { + while (!pinlist_file.eof()) { + pinlist_file.read(reinterpret_cast<char*>(&vma_start), sizeof(vma_start)); + pinlist_file.read(reinterpret_cast<char*>(&vma_length), sizeof(vma_length)); + if (pinlist_file.fail()) { return 1; } vma_start = betoh32(vma_start); vma_length = betoh32(vma_length); - pinranges.push_back(VmaRange(vma_start, vma_length)); } - close(pinlist_fd); return 0; +} + +ZipEntryCoverage PinConfigFile::to_zipfilemem(const ZipEntryInfo& info) { + ZipEntryCoverage file; + file.info = info; + + if (ranges.empty()) { + cout << "No ranges found for file " << info.name << " creating entire file range" << endl; + // Any file coming from pinconfig without explicit + // ranges will be assumed to be wanted in its entirety + ranges.push_back(VmaRange(0, info.file_size_bytes)); + } + + file.coverage.ranges = ranges; + + // Offsets specified in pinconfig file are relative to the file + // so transform to zip global offsets which are used for coverage + // computations. + file.coverage.apply_offset(info.offset_in_zip); + + file.coverage.compute_total_size(); + return file; +} + +int PinConfig::parse(std::string config_file, bool verbose) { + ifstream file(config_file); + string file_in_zip; + if (verbose) { + cout << "Parsing file: " << config_file << endl; + } + string token; + file >> token; + while (!file.eof()) { + if (token == "file") { + file >> file_in_zip; + PinConfigFile pin_config_file; + pin_config_file.filename = file_in_zip; + file >> token; + while (token != "file" && !file.eof()) { + VmaRange range; + // Inner parsing loop for per file config. + if (token == "offset") { + file >> token; + android::base::ParseUint(token, &range.offset); + file >> token; + if (token != "len") { + cerr << "Malformed file, expected 'len' after offset" << endl; + return 1; + } + file >> token; + android::base::ParseUint(token, &range.length); + pin_config_file.ranges.push_back(range); + } + file >> token; + } + files_.push_back(pin_config_file); + } else { + cerr << "Unexpected token: " << token << ". Exit read" << endl; + return 1; + } + } + + if (files_.empty()) { + cerr << "Failed parsing pinconfig file, no entries found." << endl; + return 1; + } + + if (verbose) { + cout << "Finished parsing Pinconfig file" << endl; + for (auto&& pin_file : files_) { + cout << "file=" << pin_file.filename << endl; + for (auto&& range : pin_file.ranges) { + cout << "offset=" << range.offset << " bytes=" << range.length << endl; + } + } + } + + return 0; +} + +void PinTool::set_custom_zip_inspector(ZipMemInspector* inspector) { + delete zip_inspector_; + zip_inspector_ = inspector; +} + +void PinTool::set_verbose_output(bool verbose) { + verbose_ = verbose; +} + +void PinTool::read_probe_from_pinlist(std::string custom_probe_file) { + custom_probe_file_ = custom_probe_file; + VmaRangeGroup* custom_probe = new VmaRangeGroup(); + read_pinlist_file(custom_probe_file_, custom_probe->ranges); + custom_probe->compute_total_size(); + if (custom_probe->ranges.empty()) { + cerr << "Did not find any memory range in " << custom_probe_file_ << endl; + delete custom_probe; + return; + } + zip_inspector_->set_existing_probe(custom_probe); +} + +int PinTool::probe_resident() { + return zip_inspector_->probe_resident(); +} + +void PinTool::compute_zip_entry_coverages() { + zip_inspector_->compute_per_file_coverage(); + if (verbose_) { + std::vector<ZipEntryInfo> files = zip_inspector_->get_file_infos(); + for (auto&& file : files) { + cout << "file found. name=" << file.name << " offset=" << file.offset_in_zip + << " uncompressed=" << file.uncompressed_size + << " compressed=" << file.file_size_bytes << endl + << endl; + } + } +} + +void PinTool::dump_coverages(PinTool::DumpType dump_type) { + std::vector<ZipEntryCoverage>* file_coverages; + if (dump_type == PinTool::DumpType::FILTERED) { + file_coverages = &filtered_files_; + } else if (dump_type == PinTool::DumpType::FILE_COVERAGE) { + file_coverages = &(zip_inspector_->get_file_coverages()); + } else { // PinTool::DumpType::PROBE + VmaRangeGroup* probe = zip_inspector_->get_probe(); + file_coverages = new vector<ZipEntryCoverage>(); + ZipEntryCoverage file; + file.coverage = *probe; + file.info.name = input_file_; + file.info.offset_in_zip = 0; + uint64_t file_size_bytes = get_file_size(input_file_); + if (file_size_bytes == -1) { + cerr << "Failed to dump, cannot fstat file: " << input_file_ << endl; + delete file_coverages; + return; + } + file.info.file_size_bytes = file_size_bytes; + file_coverages->push_back(file); + } + + for (auto&& file : *file_coverages) { + uint64_t total_size = file.coverage.compute_total_size(); + cout << file.info.name << " size(B)=" << file.info.file_size_bytes + << " resident(B)=" << total_size + << " resident(%)=" << (double)(total_size) / file.info.file_size_bytes * 100.0 << endl; + if (verbose_) { + cout << "file_base_zip_offset=" << file.info.offset_in_zip << endl; + } + cout << "file resident ranges" << endl; + if (dump_type != DumpType::PROBE) { + for (auto&& range : file.coverage.ranges) { + // The offset in the range represents the absolute absolute offset relative to the + // zip so substract the file base offset to get the relative offset within the file + // which may be what is worth for a user to specify in pinconfig.txt files. + uint64_t offset_in_file = range.offset - file.info.offset_in_zip; + + cout << "zip_offset=" << range.offset << " file_offset=" << offset_in_file + << " total_bytes=" << range.length << endl; + } + } else { + for (auto&& range : file.coverage.ranges) { + cout << "file_offset=" << range.offset << " total_bytes=" << range.length << endl; + } + } + cout << endl; + } + cout << endl; + if (dump_type == DumpType::PROBE) { + // For other dump types we do not create memory, we reuse from class. + delete file_coverages; + } +} + +void PinTool::filter_zip_entry_coverages(const std::string& pinconfig_filename) { + if (pinconfig_filename.length() == 0) { + // Nothing to do. + return; + } + + PinConfig* pinconfig = new PinConfig(); + if (pinconfig->parse(pinconfig_filename, verbose_) > 0) { + cerr << "Failed parsing pinconfig file " << pinconfig_filename << ". Skip filtering"; + delete pinconfig; + return; + } + + filter_zip_entry_coverages(pinconfig); +} + +void PinTool::filter_zip_entry_coverages(PinConfig* pinconfig) { + pinconfig_ = pinconfig; + + // Filter based on the per file configuration. + vector<ZipEntryCoverage> file_coverages = zip_inspector_->get_file_coverages(); + vector<ZipEntryCoverage>& filtered_files = filtered_files_; + + for (auto&& file_coverage : file_coverages) { + for (auto&& pinconfig_file : pinconfig_->files_) { + // Match each zip entry against every pattern in filter file. + std::string_view file_coverage_view(file_coverage.info.name.c_str()); + std::string_view pinconfig_view(pinconfig_file.filename.c_str()); + if (file_coverage_view.find(pinconfig_view) != std::string_view::npos) { + // Now that we found a match, create a file with offsets that are global to zip file + ZipEntryCoverage file_in_config = pinconfig_file.to_zipfilemem(file_coverage.info); + if (verbose_) { + cout << "Found a match: file=" << file_coverage.info.name + << " matching filter=" << pinconfig_file.filename << endl; + for (auto&& range : file_in_config.coverage.ranges) { + cout << "zip_offset=" << range.offset << " bytes=" << range.length << endl; + } + } + ZipEntryCoverage filtered_file = + file_coverage.compute_coverage(file_in_config.coverage); + filtered_files.push_back(filtered_file); + break; + } + } + } +} + +std::vector<ZipEntryCoverage> PinTool::get_filtered_zip_entries() { + return filtered_files_; +} + +void PinTool::write_coverages_as_pinlist(std::string output_pinlist, int64_t write_quota) { + std::vector<ZipEntryCoverage>* pinlist_coverages = nullptr; + if (!filtered_files_.empty()) { + // Highest preference is writing filtered files if they exist + if (verbose_) { + cout << "Writing pinconfig filtered file coverages" << endl; + } + pinlist_coverages = &filtered_files_; + } else if (!zip_inspector_->get_file_coverages().empty()) { + // Fallback to looking for file coverage computation + pinlist_coverages = &zip_inspector_->get_file_coverages(); + if (verbose_) { + cout << "Writing regular file coverages." << endl; + } + } + if (pinlist_coverages == nullptr) { + cerr << "Failed to find coverage to write to: " << output_pinlist << endl; + return; + } + int res = write_pinlist_file(output_pinlist, *pinlist_coverages, write_quota); + if (res > 0) { + cerr << "Failed to write pin file at: " << output_pinlist << endl; + } else { + if (verbose_) { + cout << "Finished writing pin file at: " << output_pinlist << endl; + } + } }
\ No newline at end of file diff --git a/pinner/pintool.cpp b/pinner/pintool.cpp index bdb79008..9aa47b7b 100644 --- a/pinner/pintool.cpp +++ b/pinner/pintool.cpp @@ -1,4 +1,4 @@ -#include <android-base/stringprintf.h> +#include <android-base/parseint.h> #include <fcntl.h> #include <sys/endian.h> #include <sys/mman.h> @@ -15,27 +15,82 @@ using namespace std; using namespace android::base; -enum ToolMode { PROBE, DUMP, UNKNOWN }; +enum ToolMode { + MAPPED_FILE, // Files that are mapped in memory + PINLIST, // pinlist.meta style file + UNKNOWN +}; -void print_pinner_ranges(const std::vector<VmaRange>& ranges) { - cout << "vmas to pin:" << endl; - for (int i = 0; i < ranges.size(); ++i) { - cout << StringPrintf("start=%u length=%u", ranges[i].offset, ranges[i].length) << endl; +void print_pinlist_ranges(const std::vector<VmaRange>& ranges) { + cout << "--pinlist memory ranges--" << endl; + for (auto&& range : ranges) { + cout << "start=" << range.offset << " bytes=" << range.length << endl; } } -int perform_probe(const vector<string>& options) { - std::string probed_file; - std::string output_file = "/data/local/tmp/pinlist.meta"; +void print_pinlist_summary(const std::vector<VmaRange>& ranges) { + cout << "--pinlist summary--" << endl; + uint64_t total_bytes = 0; + for (auto&& range : ranges) { + total_bytes += range.length; + } + cout << "total_bytes_to_pin=" << total_bytes << endl; +} + +int perform_file_action(const vector<string>& options) { + std::string custom_probe_file; + std::string output_file; + std::string pinconfig_file; + bool verbose = false; - int pages_per_mincore = DEFAULT_PAGES_PER_MINCORE; + bool is_zip = false; + bool dump_results = false; + ProbeType probe_type = UNSET; + int64_t write_quota = -1; // unbounded by default + + if (options.empty()) { + cerr << "Missing filename for file action, see usage for details." << endl; + return 1; + } + + std::string input_file = options[0]; + + // Validate that the file exists. + if (get_file_size(input_file) == -1) { + cerr << "Error: Could not read file: " << input_file << endl; + return 1; + } + + if (input_file.empty()) { + cerr << "Error: Should specify an input file." << endl; + return 1; + } - // Parse Args - for (int i = 0; i < options.size(); ++i) { + // Parse flags + for (int i = 1; i < options.size(); ++i) { string option = options[i]; - if (option == "-p") { + if (option == "--gen-probe") { + if (probe_type != ProbeType::UNSET) { + cerr << "Should only specify one probe treatment. See usage for details." << endl; + return 1; + } + probe_type = ProbeType::GENERATE; + continue; + } + + if (option == "--use-probe") { + if (probe_type != ProbeType::UNSET) { + cerr << "Should only specify one probe treatment. See usage for details." << endl; + return 1; + } + probe_type = ProbeType::CUSTOM; + ++i; + custom_probe_file = options[i]; + continue; + } + if (option == "--pinconfig") { ++i; - probed_file = options[i]; + pinconfig_file = options[i]; continue; } if (option == "-o") { @@ -43,125 +98,204 @@ int perform_probe(const vector<string>& options) { output_file = options[i]; continue; } + if (option == "--quota") { + ++i; + android::base::ParseInt(options[i], &write_quota); + continue; + } if (option == "-v") { verbose = true; continue; } - if (option == "-w") { - ++i; - pages_per_mincore = stoi(options[i]); + if (option == "--zip") { + is_zip = true; + continue; + } + if (option == "--dump") { + dump_results = true; continue; } } if (verbose) { - cout << "mincore window size=" << pages_per_mincore << endl; - cout << "Setting output pinlist file as " << probed_file.c_str() << endl; - cout << "Setting file to probe: " << probed_file.c_str() << endl; + cout << "Setting output pinlist file: " << output_file.c_str() << endl; + cout << "Setting input file: " << input_file.c_str() << endl; + cout << "Setting pinconfig file: " << pinconfig_file.c_str() << endl; + cout << "Setting custom probe file: " << custom_probe_file.c_str() << endl; + cout << "Setting probe type: " << probe_type << endl; + cout << "Dump enabled: " << dump_results << endl; + cout << "Is Zip file: " << is_zip << endl; + if (write_quota != -1) { + cout << "Set Write quota: " << write_quota << endl; + } } - if (probed_file.empty()) { - cerr << "Error: Should specify a file to probe."; - return 1; - } + PinTool pintool(input_file); - ResidentMemResult memresult; - int res = probe_resident_memory(probed_file, memresult, pages_per_mincore); - if (res) { - if (res == MEMINSPECT_FAIL_OPEN) { - cerr << "Failed to open file: " << probed_file << endl; + if (is_zip) { + pintool.set_verbose_output(verbose); + if (probe_type == ProbeType::CUSTOM) { + if (verbose) { + cout << "Using custom probe file: " << custom_probe_file << endl; + } + pintool.read_probe_from_pinlist(custom_probe_file); + } else if (probe_type == ProbeType::GENERATE) { + if (verbose) { + cout << "Generating probe" << endl; + } + int res = pintool.probe_resident(); + if (res > 0) { + cerr << "Failed to generate probe. Error Code: " << res << endl; + return 1; + } } + pintool.compute_zip_entry_coverages(); - if (res == MEMINSPECT_FAIL_FSTAT) { - cerr << "Failed to fstat file: " << probed_file << endl; + if (pinconfig_file.length() > 0) { + // We have provided a pinconfig file so perform filtering + // of computed coverages based on it. + pintool.filter_zip_entry_coverages(pinconfig_file); } - if (res == MEMINSPECT_FAIL_MINCORE) { - cerr << "Mincore failed for file: " << probed_file << endl; - } + if (dump_results) { + cout << endl << "----Unfiltered file coverages----" << endl << endl; + pintool.dump_coverages(PinTool::DumpType::FILE_COVERAGE); - return res; - } + if (pinconfig_file.length() > 0) { + cout << endl << "----Filtered file coverages----" << endl << endl; + pintool.dump_coverages(PinTool::DumpType::FILTERED); + } + } - cout << StringPrintf( - "Finished Probing. resident memory(KB)=%llu. file_size (KB)=%llu. " - "pin_percentage=%f", - (unsigned long long) (memresult.total_resident_bytes / 1024), (unsigned long long) (memresult.file_size_bytes / 1024), - memresult.total_resident_bytes / (float)memresult.file_size_bytes * 100) - << endl; + if (output_file.length() > 0) { + pintool.write_coverages_as_pinlist(output_file, write_quota); + } - res = write_pinlist_file(output_file, memresult.resident_memory_ranges); - if (res > 0) { - cerr << "Failed to write pin file at: " << output_file << endl; + return 0; } else { - if (verbose) { - cout << "Finished writing pin file at: " << output_file << endl; + if (probe_type != ProbeType::GENERATE) { + cerr << "Only generating probes is supported for non-zip files, please include " + "--gen-probe on your command" + << endl; + return 1; } + + // Generic file probing will just return resident memory and offsets + // without more contextual information. + VmaRangeGroup resident; + + int res = pintool.probe_resident(); + if (res > 0) { + cerr << "Failed to generate probe. Error Code: " << res << endl; + return 1; + } + + pintool.dump_coverages(PinTool::DumpType::PROBE); + + if (output_file.length() > 0) { + res = write_pinlist_file(output_file, resident.ranges, write_quota); + if (res > 0) { + cerr << "Failed to write pin file at: " << output_file << endl; + } else if (verbose) { + cout << "Finished writing pin file at: " << output_file << endl; + } + } + return res; } - return res; + return 0; } -int perform_dump(const vector<string>& options) { +int perform_pinlist_action(const vector<string>& options) { string pinner_file; bool verbose = false; - for (int i = 0; i < options.size(); ++i) { + bool dump = false; + bool summary = false; + + if (options.size() < 1) { + cerr << "Missing arguments for pinlist mode. See usage for details << endl"; + return 1; + } + pinner_file = options[0]; + for (int i = 1; i < options.size(); ++i) { string option = options[i]; - if (option == "-p") { - ++i; - pinner_file = options[i]; - continue; - } if (option == "-v") { verbose = true; } + + if (option == "--dump") { + dump = true; + } + + if (option == "--summary") { + summary = true; + } } + if (pinner_file.empty()) { cerr << "Error: Pinlist file to dump is missing. Specify it with '-p <file>'" << endl; return 1; } + if (verbose) { cout << "Setting file to dump: " << pinner_file.c_str() << endl; } + vector<VmaRange> vma_ranges; if (read_pinlist_file(pinner_file, vma_ranges) == 1) { cerr << "Failed reading pinlist file" << endl; } - print_pinner_ranges(vma_ranges); + + if (dump) { + print_pinlist_ranges(vma_ranges); + } + + if (summary) { + print_pinlist_summary(vma_ranges); + } return 0; } +void print_usage() { + const string usage = R"( + Expected usage: pintool <mode> <required> [option] + where: + ./pintool <MODE> + <MODE> + file <filename> [option] + [option] + --gen-probe + Generate a probe from current resident memory based on provided "file" + --use-probe <path_to_input_pinlist.meta> + Use a previously generated pinlist.meta style file as the probe to match against. + --dump + Dump output contents to console. + --zip + Treat the file as a zip/apk file required for doing per-file coverage analysis and generation. + --pinconfig <path_to_pinconfig.txt> + Filter output coverage ranges using a provided pinconfig.txt style file. See README.md for samples + on the format of that file. + -v + Enable verbose output. + + pinlist <pinlist_file> [option] + <pinlist_file> + this is the file that will be used for reading and it should follow the pinlist.meta format. + [option] + --dump + Dump <pinlist_file> contents to console output. + -v + Enable verbose output. + --summary + Summary results for the pinlist.meta file + )"; + cout << usage.c_str(); +} + int main(int argc, char** argv) { if (argc == 1) { - const string usage = R"( -Expected usage: pintool <mode> <required> [option] -where: -<file_to_pin> is a file currently mapped by another process and in memory. -<mode> : - probe - This mode will probe resident memory for a file and generate a pinlist.meta file - that can be interpreted by the PinnerService. - - <required> - -p <file_to_probe> - This option will probe the specified file - [option]: - -o <file> - Specify the output file for the pinlist file. - (default=/data/local/tmp/pinlist.meta) - -v - Enable verbose output. - -w - Mincore total pages per mincore window. Bigger windows - will use more memory but may be slightly faster. (default=1) - dump - <required> - -p <input_pinlist_file> - Specify the input pinlist file to dump -)"; - - cout << usage.c_str(); + print_usage(); return 0; } @@ -169,14 +303,20 @@ where: cerr << "<mode> is missing"; return 1; } - ToolMode mode = UNKNOWN; - if (strcmp(argv[1], "probe") == 0) { - mode = PROBE; - } else if (strcmp(argv[1], "dump") == 0) { - mode = DUMP; + + if (strcmp(argv[1], "--help") == 0) { + print_usage(); + return 0; + } + + ToolMode mode = ToolMode::UNKNOWN; + if (strcmp(argv[1], "file") == 0) { + mode = ToolMode::MAPPED_FILE; + } else if (strcmp(argv[1], "pinlist") == 0) { + mode = ToolMode::PINLIST; } - if (mode == UNKNOWN) { + if (mode == ToolMode::UNKNOWN) { cerr << "Failed to find mode: " << argv[1] << ". See usage for available modes." << endl; return 1; } @@ -188,15 +328,15 @@ where: int res; switch (mode) { - case PROBE: - res = perform_probe(options); + case ToolMode::MAPPED_FILE: + res = perform_file_action(options); break; - case DUMP: - res = perform_dump(options); + case ToolMode::PINLIST: + res = perform_pinlist_action(options); break; - case UNKNOWN: + case ToolMode::UNKNOWN: + cerr << "Unknown <MODE> see usage for details." << endl; return 1; - break; } return res; diff --git a/pinner/sample_pinconfig.txt b/pinner/sample_pinconfig.txt new file mode 100644 index 00000000..3d9a13a6 --- /dev/null +++ b/pinner/sample_pinconfig.txt @@ -0,0 +1,5 @@ +file lib/arm64-v8a/libcrashpad_handler_trampoline.so +file libmonochrome_64.so +offset 108391596 +len 33348032 +file libarcore_sdk_c.so
\ No newline at end of file diff --git a/pinner/tests/Android.bp b/pinner/tests/Android.bp index 8c3f262a..f7b04d95 100644 --- a/pinner/tests/Android.bp +++ b/pinner/tests/Android.bp @@ -18,15 +18,21 @@ package { cc_test { name: "meminspect_tests", + + test_suites: ["device-tests"], + + // Required for reading-writing files which are part of the tests. + require_root: true, + shared_libs: [ "libbase", + "libziparchive", ], cppflags: [ "-g", "-Wall", "-Werror", - "-Wno-missing-field-initializers", ], static_libs: [ @@ -43,7 +49,7 @@ cc_test { "-Wall", "-Wextra", "-Werror", - "-O0", + "-O0", // as some tests rely on compiler keeping code as is ], compile_multilib: "first", @@ -52,15 +58,20 @@ cc_test { cc_test { name: "pintool_tests", + test_suites: ["device-tests"], + + // Required for reading-writing files which are part of the tests. + require_root: true, + shared_libs: [ "libbase", + "libziparchive", ], cppflags: [ "-g", "-Wall", "-Werror", - "-Wno-missing-field-initializers", ], static_libs: [ @@ -77,7 +88,7 @@ cc_test { "-Wall", "-Wextra", "-Werror", - "-O0", + "-O0", // as some tests rely on compiler keeping code as is ], compile_multilib: "first", diff --git a/pinner/tests/TEST_MAPPING b/pinner/tests/TEST_MAPPING new file mode 100644 index 00000000..73af05f1 --- /dev/null +++ b/pinner/tests/TEST_MAPPING @@ -0,0 +1,10 @@ +{ + "postsubmit": [ + { + "name": "pintool_tests" + }, + { + "name": "meminspect_tests" + } + ] +}
\ No newline at end of file diff --git a/pinner/tests/meminspect_tests.cpp b/pinner/tests/meminspect_tests.cpp index da3826db..98c51def 100644 --- a/pinner/tests/meminspect_tests.cpp +++ b/pinner/tests/meminspect_tests.cpp @@ -74,15 +74,15 @@ TEST(meminspect_test, inspect_matches_resident) { return; } - ResidentMemResult vmas_resident; + VmaRangeGroup vmas_resident; int res = probe_resident_memory(test_file, vmas_resident, 1); EXPECT_TRUE(res == 0); // Probing the file without reading anything yields no resident memory - EXPECT_TRUE(vmas_resident.resident_memory_ranges.empty()); + EXPECT_TRUE(vmas_resident.ranges.empty()); // Clear our to start fresh for next probe. - vmas_resident = ResidentMemResult(); + vmas_resident = VmaRangeGroup(); int pages_to_read = 1; char* read_data = new char[pages_to_read]; @@ -94,12 +94,191 @@ TEST(meminspect_test, inspect_matches_resident) { EXPECT_TRUE(res == 0); // The amount of memory paged in is outside our control, but we should have some. - EXPECT_TRUE(vmas_resident.total_resident_bytes > 0); - EXPECT_TRUE(vmas_resident.resident_memory_ranges.size() == 1); - EXPECT_TRUE(vmas_resident.resident_memory_ranges[0].offset == 0); - EXPECT_TRUE((uint64_t)vmas_resident.resident_memory_ranges[0].length == - vmas_resident.total_resident_bytes); + uint64_t resident_total_size = vmas_resident.compute_total_size(); + EXPECT_TRUE(resident_total_size > 0); + EXPECT_TRUE(vmas_resident.ranges.size() == 1); + EXPECT_TRUE(vmas_resident.ranges[0].offset == 0); + EXPECT_TRUE((uint64_t)vmas_resident.ranges[0].length == resident_total_size); close(test_file_fd); remove(test_file.c_str()); +} + +TEST(meminspect_test, custom_probe_coverage_matches_with_probe) { + ZipMemInspector inspector(""); + VmaRangeGroup* probe = new VmaRangeGroup(); + probe->ranges.push_back(VmaRange(0, 500)); + probe->ranges.push_back(VmaRange(700, 100)); + probe->ranges.push_back(VmaRange(1000, 500)); + probe->ranges.push_back(VmaRange(2000, 100)); + // Probed Resident Memory Offset ranges: + // [0,500],[700,800],[1000,1500],[2000,2100] + EXPECT_EQ(probe->compute_total_size(), (unsigned long long)1200); + inspector.set_existing_probe(probe); + + // Emulate reading some files from the zip to compute their coverages + // fake1 memory offset ranges [100,300] + ZipEntryInfo info; + info.name = "fake1"; + info.offset_in_zip = 100; + info.file_size_bytes = 200; + inspector.add_file_info(info); + + // fake2 memory offset ranges [600,1200] + ZipEntryInfo info2; + info2.name = "fake2"; + info2.offset_in_zip = 600; + info2.file_size_bytes = 600; + inspector.add_file_info(info2); + + inspector.compute_per_file_coverage(); + std::vector<ZipEntryCoverage> coverages = inspector.get_file_coverages(); + EXPECT_EQ(coverages.size(), (size_t)2); + + // Result coverage for fake1 should be: [100,300] + EXPECT_EQ(coverages[0].coverage.ranges[0].offset, (uint32_t)100); + EXPECT_EQ(coverages[0].coverage.ranges[0].length, (uint32_t)200); + EXPECT_EQ(coverages[0].coverage.compute_total_size(), (unsigned long long)200); + EXPECT_EQ(coverages[0].info.name, "fake1"); + EXPECT_EQ(coverages[0].info.offset_in_zip, (uint32_t)100); + EXPECT_EQ(coverages[0].info.file_size_bytes, (uint32_t)200); + + // coverage coverage for fake2 should be: [700,800] and [1000,1200] + EXPECT_EQ(coverages[1].coverage.ranges[0].offset, (uint32_t)700); + EXPECT_EQ(coverages[1].coverage.ranges[0].length, (uint32_t)100); + EXPECT_EQ(coverages[1].coverage.ranges[1].offset, (uint32_t)1000); + EXPECT_EQ(coverages[1].coverage.ranges[1].length, (uint32_t)200); + EXPECT_EQ(coverages[1].coverage.compute_total_size(), (unsigned long long)300); // 100 + + // 200 + EXPECT_EQ(coverages[1].info.name, "fake2"); + EXPECT_EQ(coverages[1].info.offset_in_zip, (uint32_t)600); + EXPECT_EQ(coverages[1].info.file_size_bytes, (uint32_t)600); +} + +TEST(meminspect_test, whole_file_coverage_against_probe) { + ZipMemInspector inspector(""); + + // Emulate reading some files from the zip to compute their coverages + // fake1 memory offset ranges [100,300] + ZipEntryInfo info; + info.name = "fake1"; + info.offset_in_zip = 100; + info.file_size_bytes = 200; + inspector.add_file_info(info); + + // fake2 memory offset ranges [600,1200] + ZipEntryInfo info2; + info2.name = "fake2"; + info2.offset_in_zip = 600; + info2.file_size_bytes = 600; + inspector.add_file_info(info2); + + inspector.compute_per_file_coverage(); + std::vector<ZipEntryCoverage> coverages = inspector.get_file_coverages(); + EXPECT_EQ(coverages.size(), (size_t)2); + + // Check that coverage matches entire file sizes + EXPECT_EQ(coverages[0].coverage.ranges[0].offset, (uint32_t)100); + EXPECT_EQ(coverages[0].coverage.ranges[0].length, (uint32_t)200); + EXPECT_EQ(coverages[0].coverage.compute_total_size(), (unsigned long long)200); + EXPECT_EQ(coverages[0].info.name, "fake1"); + EXPECT_EQ(coverages[0].info.offset_in_zip, (uint32_t)100); + EXPECT_EQ(coverages[0].info.file_size_bytes, (uint32_t)200); + + EXPECT_EQ(coverages[1].coverage.ranges[0].offset, (uint32_t)600); + EXPECT_EQ(coverages[1].coverage.ranges[0].length, (uint32_t)600); + EXPECT_EQ(coverages[1].coverage.compute_total_size(), (unsigned long long)600); + EXPECT_EQ(coverages[1].info.name, "fake2"); + EXPECT_EQ(coverages[1].info.offset_in_zip, (uint32_t)600); + EXPECT_EQ(coverages[1].info.file_size_bytes, (uint32_t)600); +} + +TEST(meminspect_test, file_multiple_ranges_matches_probe) { + VmaRangeGroup probe; + probe.ranges.push_back(VmaRange(0, 500)); + probe.ranges.push_back(VmaRange(700, 100)); + probe.ranges.push_back(VmaRange(1000, 500)); + probe.ranges.push_back(VmaRange(2000, 100)); + // Probed Resident Memory Offset ranges: + // [0,500],[700,800],[1000,1500],[2000,2100] + EXPECT_EQ(probe.compute_total_size(), (unsigned long long)1200); + + std::vector<ZipEntryCoverage> desired_coverages; + + // fake1 file resides between [100,1100] + // desired ranges are [100,200],[400,710],[820,850] + ZipEntryCoverage file1_mem; + file1_mem.info.name = "fake1"; + file1_mem.info.offset_in_zip = 100; + file1_mem.info.file_size_bytes = 1000; + file1_mem.coverage.ranges.push_back(VmaRange(100, 100)); + file1_mem.coverage.ranges.push_back(VmaRange(400, 310)); + file1_mem.coverage.ranges.push_back(VmaRange(820, 30)); + desired_coverages.push_back(file1_mem); + + // fake2 memory offset ranges [1300,2100] + // desired ranges are [1400,1500],[1600,1650],[1800,2050] + ZipEntryCoverage file2_mem; + file2_mem.info.name = "fake2"; + file2_mem.info.offset_in_zip = 1300; + file2_mem.info.file_size_bytes = 750; + file2_mem.coverage.ranges.push_back(VmaRange(1400, 100)); + file2_mem.coverage.ranges.push_back(VmaRange(1600, 50)); + file2_mem.coverage.ranges.push_back(VmaRange(1800, 250)); + desired_coverages.push_back(file2_mem); + + std::vector<ZipEntryCoverage> coverages = + ZipMemInspector::compute_coverage(desired_coverages, &probe); + + EXPECT_EQ(coverages.size(), (size_t)2); + + // Result coverage for fake1 should be: [100,200],[400,500],[700,710] + EXPECT_EQ(coverages[0].coverage.ranges[0].offset, (uint32_t)100); + EXPECT_EQ(coverages[0].coverage.ranges[0].length, (uint32_t)100); + EXPECT_EQ(coverages[0].coverage.ranges[1].offset, (uint32_t)400); + EXPECT_EQ(coverages[0].coverage.ranges[1].length, (uint32_t)100); + EXPECT_EQ(coverages[0].coverage.ranges[2].offset, (uint32_t)700); + EXPECT_EQ(coverages[0].coverage.ranges[2].length, (uint32_t)10); + + EXPECT_EQ(coverages[0].coverage.compute_total_size(), (unsigned long long)210); + EXPECT_EQ(coverages[0].info.name, "fake1"); + EXPECT_EQ(coverages[0].info.offset_in_zip, (uint32_t)100); + EXPECT_EQ(coverages[0].info.file_size_bytes, (uint32_t)1000); + + // coverage coverage for fake2 should be: [1400,1500],[2000,2050] + EXPECT_EQ(coverages[1].coverage.ranges[0].offset, (uint32_t)1400); + EXPECT_EQ(coverages[1].coverage.ranges[0].length, (uint32_t)100); + EXPECT_EQ(coverages[1].coverage.ranges[1].offset, (uint32_t)2000); + EXPECT_EQ(coverages[1].coverage.ranges[1].length, (uint32_t)50); + EXPECT_EQ(coverages[1].coverage.compute_total_size(), (unsigned long long)150); + EXPECT_EQ(coverages[1].info.name, "fake2"); + EXPECT_EQ(coverages[1].info.offset_in_zip, (uint32_t)1300); + EXPECT_EQ(coverages[1].info.file_size_bytes, (uint32_t)750); +} + +TEST(meminspect_test, range_alignment_and_merge_matches) { + ZipMemInspector inspector(""); + VmaRangeGroup* probe = new VmaRangeGroup(); + probe->ranges.push_back(VmaRange(0, 500)); + probe->ranges.push_back(VmaRange(700, 100)); + int page_size = 4096; + + // Probed Resident Memory Offset ranges: + // [0,500],[700,800] + inspector.set_existing_probe(probe); + + // When we page align, we should end up with [0,500],[0,800] + align_ranges(probe->ranges, page_size); + EXPECT_EQ(probe->ranges[0].offset, (uint32_t)0); + EXPECT_EQ(probe->ranges[0].length, (uint32_t)500); + EXPECT_EQ(probe->ranges[1].offset, (uint32_t)0); + EXPECT_EQ(probe->ranges[1].length, (uint32_t)800); + EXPECT_EQ(probe->ranges.size(), (uint32_t)2); + + // Because we have overlapping ranges, a union-merge should + // skip duplication of intersections and end up with [0,800] + std::vector<VmaRange> merged = merge_ranges(probe->ranges); + EXPECT_EQ(merged[0].offset, (uint32_t)0); + EXPECT_EQ(merged[0].length, (uint32_t)800); + EXPECT_EQ(merged.size(), (uint32_t)1); }
\ No newline at end of file diff --git a/pinner/tests/pintool_tests.cpp b/pinner/tests/pintool_tests.cpp index b76c5284..f351a90a 100644 --- a/pinner/tests/pintool_tests.cpp +++ b/pinner/tests/pintool_tests.cpp @@ -1,5 +1,6 @@ #include <gtest/gtest.h> #include <stdio.h> +#include <list> #include <pin_utils.h> @@ -7,10 +8,11 @@ using namespace std; TEST(pintool_test, pinlist_matches_memranges) { vector<VmaRange> vma_ranges; + unsigned int page_size = sysconf(_SC_PAGESIZE); vma_ranges.push_back(VmaRange(0, 500)); - vma_ranges.push_back(VmaRange(1000, 5500)); - vma_ranges.push_back(VmaRange(10000, 13000)); - vma_ranges.push_back(VmaRange(26000, 35000)); + vma_ranges.push_back(VmaRange(5000, 5500)); + vma_ranges.push_back(VmaRange(21000, 13000)); + vma_ranges.push_back(VmaRange(50000, 35000)); string test_file = "/data/local/tmp/pintool_test"; write_pinlist_file(test_file, vma_ranges); @@ -20,9 +22,116 @@ TEST(pintool_test, pinlist_matches_memranges) { EXPECT_EQ(vma_ranges.size(), read_ranges.size()); for (size_t i = 0; i < vma_ranges.size(); ++i) { - EXPECT_EQ(vma_ranges[i].offset, read_ranges[i].offset); - EXPECT_EQ(vma_ranges[i].length, read_ranges[i].length); + // We expect to write pinlists that are page-aligned, so + // we compare against page aligned offsets. + uint64_t unaligned_bytes = vma_ranges[i].offset % page_size; + EXPECT_EQ(vma_ranges[i].offset - unaligned_bytes, read_ranges[i].offset); + EXPECT_EQ(vma_ranges[i].length + unaligned_bytes, read_ranges[i].length); } remove(test_file.c_str()); +} + +TEST(pintool_test, pinlist_quota_applied) { + vector<VmaRange> vma_ranges; + unsigned int page_size = sysconf(_SC_PAGESIZE); + vma_ranges.push_back(VmaRange(0, 100)); + vma_ranges.push_back(VmaRange(page_size, 500)); + vma_ranges.push_back(VmaRange(page_size * 2, 300)); + vma_ranges.push_back(VmaRange(page_size * 3, 200)); + + const int ranges_to_write = 700; + string test_file = "/data/local/tmp/pintool_test"; + write_pinlist_file(test_file, vma_ranges, ranges_to_write); + + vector<VmaRange> read_ranges; + read_pinlist_file(test_file, read_ranges); + + int total_length = 0; + for (size_t i = 0; i < read_ranges.size(); ++i) { + total_length += read_ranges[i].length; + } + EXPECT_EQ(total_length, ranges_to_write); + + remove(test_file.c_str()); +} + +TEST(pintool_test, pinconfig_filter_coverage_matches) { + VmaRangeGroup* probe = new VmaRangeGroup(); + probe->ranges.push_back(VmaRange(0, 500)); + probe->ranges.push_back(VmaRange(1000, 5000)); + + ZipMemInspector* inspector = new ZipMemInspector(""); + + // Probed Resident Memory Offset ranges: + // [0,500],[1000,6000] + probe->compute_total_size(); + inspector->set_existing_probe(probe); + + // Emulate reading some files from the zip to compute their coverages + // fake1 memory offset ranges [100,400] + ZipEntryInfo info; + info.name = "fake1"; + info.offset_in_zip = 100; + info.file_size_bytes = 300; + inspector->add_file_info(info); + + // fake2 memory offset ranges [600,3000] + ZipEntryInfo info2; + info2.name = "fake2"; + info2.offset_in_zip = 600; + info2.file_size_bytes = 2400; + inspector->add_file_info(info2); + + ZipEntryInfo info3; + info2.name = "fake3"; + info2.offset_in_zip = 3100; + info2.file_size_bytes = 200; + inspector->add_file_info(info3); + + // Create a fake pinconfig file + PinConfig* pinconfig = new PinConfig(); + + // First file we want it entirely so don't provide ranges + PinConfigFile pinconfig_file_1; + pinconfig_file_1.filename = "fake1"; + pinconfig->files_.push_back(pinconfig_file_1); + + // Add a partially matched file + PinConfigFile pinconfig_file_2; + pinconfig_file_2.filename = "fake2"; + pinconfig_file_2.ranges.push_back(VmaRange(100, 500)); + pinconfig_file_2.ranges.push_back(VmaRange(800, 200)); + pinconfig->files_.push_back(pinconfig_file_2); + + // Add a file that does not exist + PinConfigFile pinconfig_file_3; + pinconfig_file_3.filename = "fake4"; + pinconfig_file_3.ranges.push_back(VmaRange(0, 1000)); + pinconfig->files_.push_back(pinconfig_file_3); + + PinTool pintool(""); + pintool.set_custom_zip_inspector(inspector); + pintool.compute_zip_entry_coverages(); + pintool.filter_zip_entry_coverages(pinconfig); + + std::vector<ZipEntryCoverage> filtered = pintool.get_filtered_zip_entries(); + + // We only matched 2 files, one should not have matched to any filter. + EXPECT_EQ(filtered.size(), (unsigned long)2); + EXPECT_EQ(filtered[0].info.name, "fake1"); + EXPECT_EQ(filtered[0].coverage.ranges[0].offset, (unsigned long)100); + EXPECT_EQ(filtered[0].coverage.ranges[0].length, (unsigned long)300); + + // Probe Resident has [0,500],[1000,6000]. + // fake2 file lives within [600,3000] + // fake2 relative offsets from pinconfig [100,600],[800,1000] + // fake2 absolute zip offsets are [700,1200],[1400,1600] + // then matching absolute offsets against resident yields [1000,1200],[1400,1600] + EXPECT_EQ(filtered[1].info.name, "fake2"); + EXPECT_EQ(filtered[1].info.offset_in_zip, (unsigned long)600); + EXPECT_EQ(filtered[1].coverage.ranges[0].offset, (unsigned long)1000); + EXPECT_EQ(filtered[1].coverage.ranges[0].length, (unsigned long)200); + EXPECT_EQ(filtered[1].coverage.ranges[1].offset, (unsigned long)1400); + EXPECT_EQ(filtered[1].coverage.ranges[1].length, (unsigned long)200); }
\ No newline at end of file |