summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorAndroid Build Coastguard Worker <android-build-coastguard-worker@google.com>2024-03-12 23:54:17 +0000
committerGerrit Code Review <noreply-gerritcodereview@google.com>2024-03-12 23:54:17 +0000
commitf5a14a11a6146cb9c3e2801737c23681478b607d (patch)
tree61ae2e0c9a640a74304892130b7c5c764a5c8102
parent3b822f611afbbcc1e62c3110daff07898c23fc34 (diff)
parentf36506d482269ceb2533075a47ba2e72c6497cb4 (diff)
downloadextras-f5a14a11a6146cb9c3e2801737c23681478b607d.tar.gz
Merge "Snap for 11566117 from 0a775d8aead45c06dc1d2238901d5f4dc4089258 to sdk-release" into sdk-releaseplatform-tools-35.0.1
-rw-r--r--kcmdlinectrl/kcmdlinectrl.cc28
-rw-r--r--[-rwxr-xr-x]pagecache/pagecache.py5
-rw-r--r--pinner/Android.bp55
-rw-r--r--pinner/OWNERS2
-rw-r--r--pinner/README.md144
-rw-r--r--pinner/include/meminspect.h268
-rw-r--r--pinner/include/pin_utils.h116
-rw-r--r--pinner/meminspect.cpp324
-rw-r--r--pinner/pin_utils.cpp347
-rw-r--r--pinner/pintool.cpp343
-rw-r--r--pinner/sample_pinconfig.txt5
-rw-r--r--pinner/tests/Android.bp95
-rw-r--r--pinner/tests/TEST_MAPPING10
-rw-r--r--pinner/tests/meminspect_tests.cpp284
-rw-r--r--pinner/tests/pintool_tests.cpp137
-rw-r--r--profcollectd/libprofcollectd/config.rs3
16 files changed, 2148 insertions, 18 deletions
diff --git a/kcmdlinectrl/kcmdlinectrl.cc b/kcmdlinectrl/kcmdlinectrl.cc
index 5dd55af6..aafdb9a8 100644
--- a/kcmdlinectrl/kcmdlinectrl.cc
+++ b/kcmdlinectrl/kcmdlinectrl.cc
@@ -27,12 +27,10 @@
#include <functional>
#include <iostream>
-using namespace std;
-
void PrintUsage(const char* progname) {
- std::cerr << "USAGE: " << progname << " get [PROPERTY]" << endl;
- std::cerr << " " << progname << " store [PROPERTY] [VALUE]" << endl;
- std::cerr << " " << progname << " update-props" << endl;
+ std::cerr << "USAGE: " << progname << " get [PROPERTY]" << std::endl;
+ std::cerr << " " << progname << " store [PROPERTY] [VALUE]" << std::endl;
+ std::cerr << " " << progname << " update-props" << std::endl;
}
int UpdateProps() {
@@ -40,7 +38,7 @@ int UpdateProps() {
.magic = MISC_KCMDLINE_MAGIC_HEADER};
std::string err;
if (!ReadMiscKcmdlineMessage(&m, &err)) {
- LOG(ERROR) << "Failed to read from misc: " << err << endl;
+ LOG(ERROR) << "Failed to read from misc: " << err;
return 1;
}
@@ -64,12 +62,12 @@ int PrintProperty(const char* property_name) {
std::string err;
if (!ReadMiscKcmdlineMessage(&m, &err)) {
- LOG(ERROR) << "Failed to read from misc: " << err << endl;
+ LOG(ERROR) << "Failed to read from misc: " << err;
return 1;
}
if (m.magic != MISC_KCMDLINE_MAGIC_HEADER || m.version != MISC_KCMDLINE_MESSAGE_VERSION) {
- cout << "kcmdline message is invalid, treating all flags as zero" << endl;
+ std::cout << "kcmdline message is invalid, treating all flags as zero" << std::endl;
m = {.version = MISC_KCMDLINE_MESSAGE_VERSION,
.magic = MISC_KCMDLINE_MAGIC_HEADER,
.kcmdline_flags = 0};
@@ -78,10 +76,10 @@ int PrintProperty(const char* property_name) {
if (!strcmp(property_name, "binder")) {
bool use_rust_binder = (m.kcmdline_flags & MISC_KCMDLINE_BINDER_RUST) != 0;
const char* binder_value = use_rust_binder ? "rust" : "c";
- cout << "binder=" << binder_value << endl;
+ std::cout << "binder=" << binder_value << std::endl;
return 0;
} else {
- LOG(ERROR) << "Unknown property name: " << property_name << endl;
+ LOG(ERROR) << "Unknown property name: " << property_name;
return 1;
}
}
@@ -92,12 +90,12 @@ int StoreProperty(const char* property_name, const char* new_value) {
std::string err;
if (!ReadMiscKcmdlineMessage(&m, &err)) {
- LOG(ERROR) << "Failed to read from misc: " << err << endl;
+ LOG(ERROR) << "Failed to read from misc: " << err;
return 1;
}
if (m.magic != MISC_KCMDLINE_MAGIC_HEADER || m.version != MISC_KCMDLINE_MESSAGE_VERSION) {
- cout << "kcmdline message is invalid, resetting it" << endl;
+ std::cout << "kcmdline message is invalid, resetting it" << std::endl;
m = {.version = MISC_KCMDLINE_MESSAGE_VERSION,
.magic = MISC_KCMDLINE_MAGIC_HEADER,
.kcmdline_flags = 0};
@@ -109,16 +107,16 @@ int StoreProperty(const char* property_name, const char* new_value) {
} else if (!strcmp(new_value, "c")) {
m.kcmdline_flags &= !MISC_KCMDLINE_BINDER_RUST;
} else {
- LOG(ERROR) << "Binder property can only by 'c' or 'rust', but got " << new_value << endl;
+ LOG(ERROR) << "Binder property can only be 'c' or 'rust', but got " << new_value;
return 1;
}
} else {
- LOG(ERROR) << "Unknown property name: " << property_name << endl;
+ LOG(ERROR) << "Unknown property name: " << property_name;
return 1;
}
if (!WriteMiscKcmdlineMessage(m, &err)) {
- LOG(ERROR) << "Failed to write to misc: " << err << endl;
+ LOG(ERROR) << "Failed to write to misc: " << err;
return 1;
}
diff --git a/pagecache/pagecache.py b/pagecache/pagecache.py
index cb9c7396..3f96a5d1 100755..100644
--- a/pagecache/pagecache.py
+++ b/pagecache/pagecache.py
@@ -261,8 +261,9 @@ def get_inode_data(datafile, dumpfile, adb_serial):
else:
# Build inode maps if we were tracing page cache
print('Downloading inode data from device')
- stat_dump = AdbUtils.do_preprocess_adb_cmd('find /system /data /vendor ' +
- '-exec stat -c "%d %i %s %n" {} \;', adb_serial)
+ stat_dump = AdbUtils.do_preprocess_adb_cmd(
+ 'find /apex /system /system_ext /product /data /vendor ' +
+ '-exec stat -c "%d %i %s %n" {} \;', adb_serial)
if stat_dump is None:
print 'Could not retrieve inode data from device.'
sys.exit(1)
diff --git a/pinner/Android.bp b/pinner/Android.bp
new file mode 100644
index 00000000..40197ba2
--- /dev/null
+++ b/pinner/Android.bp
@@ -0,0 +1,55 @@
+//
+// Copyright (C) 2017 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package {
+ default_applicable_licenses: ["Android-Apache-2.0"],
+}
+
+cc_library_static {
+ name: "libmeminspect",
+ srcs: [
+ "meminspect.cpp",
+ "pin_utils.cpp"
+ ],
+ shared_libs: [
+ "libbase",
+ "libziparchive",
+ ],
+ export_shared_lib_headers: ["libziparchive"],
+ export_include_dirs: ["include"],
+ cppflags: [
+ "-g",
+ "-Wall",
+ "-Werror",
+ ],
+}
+
+cc_binary {
+ name: "pintool",
+ srcs: ["pintool.cpp"],
+ shared_libs: [
+ "libbase",
+ "libziparchive",
+ ],
+ static_libs: [
+ "libmeminspect",
+ ],
+ cppflags: [
+ "-g",
+ "-Wall",
+ "-Werror",
+ ],
+}
diff --git a/pinner/OWNERS b/pinner/OWNERS
new file mode 100644
index 00000000..0636f908
--- /dev/null
+++ b/pinner/OWNERS
@@ -0,0 +1,2 @@
+edgararriaga@google.com
+shayba@google.com \ No newline at end of file
diff --git a/pinner/README.md b/pinner/README.md
new file mode 100644
index 00000000..2115e809
--- /dev/null
+++ b/pinner/README.md
@@ -0,0 +1,144 @@
+# Pin Tool
+
+This tool is currently used mainly for:
+
+1) Inspecting resident memory and locality
+2) Generating and inspecting pinlist.meta used by Android's PinnerService
+
+For memory inspection, it allows probing live memory and providing the memory
+locations that are resident which can be used for diagnosis or as part of PGO
+optimizations.
+
+## Build and Install the tool
+
+To build and push the binary to device
+```
+mm pintool
+and push $ANDROID_REPO/out/target/product/<lunchtarget>/system/bin/pintool /data/local/tmp/pintool
+adb shell
+cd /data/local/tmp/pintool
+```
+
+
+## How to use the tool to generate pinner files
+
+Here are some sample use cases for this tool.
+
+### Sample Use Case 1: Probe Resident Memory for a mapped file or library and dump to console
+
+Executing the following command and providing the path to a library mapped by a process
+it will dump the resident memory ranges.
+
+```
+./pintool file <path_to_your_library> --gen-probe --dump
+```
+
+Note: you can use any kind of mapped file such as .so, .apk, etc.
+
+### Sample Use Case 2: Probe per-file Resident Memory for a mapped apk file and dump to console
+
+Executing this command will inspect resident memory for a zip file and dump
+per-file breakdowns.
+
+```
+./pintool file <path_to_myfile.apk> --zip --gen-probe --dump
+```
+
+### Sample Use Case 3: Probe per-file resident memory for a mapped apk, dump and generate pinlist.meta
+```
+./pintool file <path_to_myfile.apk> --zip --gen-probe --dump -o pinlist.meta
+```
+
+### Sample Use Case 4: Probe per-file resident memory and filter it with a provided pinconfig
+```
+./pintool file <path_to_myfile.apk> --zip --gen-probe --pinconfig pinconfig.txt --dump -o pinlist.meta
+```
+
+### Sample Use Case 5: Dump contents of a provided pinlist.meta file
+```
+./pintool pinlist pinlist.meta --dump -v
+```
+
+### Sample Use Case 6: Read a zip and filter based on a pinconfig file and generate pinlist.meta without probing
+
+This will skip doing any probing and it will just apply filtering based on the pinconfig.txt, this is helpful
+in cases where you do not intend to do any kind of PGO probing and know exactly what ranges you want to pin within your file
+
+```
+./pintool file <path_to_myfile.apk> --zip --pinconfig pinconfig.txt --dump -o pinlist.meta
+```
+
+### Sample Use Case 7: Load an existing zip probe and inspect its per-file contents
+
+```
+./pintool file /data/app/~~tmTrs5_XINwbpYWroRu5rA==/org.chromium.trichromelibrary_602300034-EFoOwMgVNBbwkMnp9zcWbg==/base.apk --zip --use-probe pinlist.meta --dump
+```
+
+
+## Pinconfig File Structure
+
+Pinconfig files specify a custom filter to be applied on top of a generated or provided memory probe
+it should specify a subset of files and optionally ranges within those files to be matched against
+and subsequently kept in case a pinlist.meta is generated.
+
+A `pinconfig.txt` is just a list of files with a key value pair separated by a newline.
+
+`pinconfig.txt` structure pattern:
+```
+(file <file>
+[offset <value>
+length <value>])*
+```
+where:
+<file>
+ Filename as a string, the parser will do a contains like operation (e.g. GLOB(*<file>*)) to match against files
+ within the zip file and stop on first match.
+<value>
+ Unsigned integer value
+
+Note: `offset` and `length` tokens are optional and if ommited, the whole file will be considered desired.
+
+
+Example `pinconfig.txt`:
+```
+file lib/arm64-v8a/libcrashpad_handler_trampoline.so
+file libmonochrome_64.so
+offset 1000
+len 50000
+file libarcore_sdk_c.so
+```
+
+## Pinlist.meta files
+
+"pinlist.meta" files are consumed by PinnerService.
+
+These files specify a list of memory ranges to be pinned (mlocked).
+If Android's PinnerService allows your app pinning, it will read the pinlist.meta
+file from inside your apk's assets folder (assets/pinlist.meta) and pin based
+on the specified ranges.
+
+Note: The PinnerService will need to support pinning your apk in order for the
+pinlist.meta file to be used.
+
+A pinlist.meta file is a binary file with a set of tuples of OFFSET and LENGTH
+stored in Big Endian format.
+
+4 byte: OFFSET
+4 byte: LEN
+
+pinlist.meta
+```
+OFFSET LEN*
+```
+
+So to read those files, it is usually helpful to use the `pintool`.
+
+## Other potential uses
+
+Outside of pinner service, the tool can be used to inspect resident memory for
+any file in memory.
+
+## Extra information
+
+the pinlist.meta depends on the apk contents and needs to be regenrated if
+you are pushing a new version of your apk. \ No newline at end of file
diff --git a/pinner/include/meminspect.h b/pinner/include/meminspect.h
new file mode 100644
index 00000000..a2e09ec2
--- /dev/null
+++ b/pinner/include/meminspect.h
@@ -0,0 +1,268 @@
+#pragma once
+
+#include <android-base/stringprintf.h>
+#include <fcntl.h>
+#include <sys/endian.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <iostream>
+#include <string>
+#include <vector>
+#include "ziparchive/zip_archive.h"
+
+#define MEMINSPECT_FAIL_OPEN 1
+#define MEMINSPECT_FAIL_FSTAT 2
+#define MEMINSPECT_FAIL_MINCORE 3
+
+#define DEFAULT_PAGES_PER_MINCORE 1
+
+/**
+ * This class stores an offset defined vma which exists
+ * relative to another memory address.
+ */
+class VmaRange {
+ public:
+ uint32_t offset;
+ uint32_t length;
+
+ VmaRange() {}
+ VmaRange(uint32_t off, uint32_t len) : offset(off), length(len) {}
+
+ bool is_empty() const;
+
+ /**
+ * @brief Compute the intersection of this range with another range
+ *
+ * Intersection Operation:
+ *
+ * Example 1:
+ * [ Range A ]
+ * [ Range B ]
+ * Intersection:
+ * [ C ]
+ *
+ * Example 2:
+ * [ Range A ] [ Range B ]
+ * No Intersection
+ *
+ * @param target range to test against
+ * @return the intersection range, if none is found, empty range is returned.
+ */
+ VmaRange intersect(const VmaRange& target) const;
+
+ /**
+ * @brief Merges the current range with a target range using a union operation
+ * that is only successful when overlapping ranges occur.
+ * A visual explanation can be seen as:
+ *
+ * Union-merge Operation:
+ *
+ * Example 1:
+ * [ Range A ]
+ * [ Range B ]
+ * Merged:
+ * [ Range C ]
+ *
+ * Example 2:
+ * [ Range A ] [ Range B ]
+ * Fails, no merge available.
+ *
+ * @param target The range to test against.
+ * @param result Upon successfully merging, contains the resulting range.
+ * @return the merged range, if none is found, empty range is returned.
+ */
+ VmaRange union_merge(const VmaRange& target) const;
+
+ uint32_t end_offset() const;
+};
+
+/**
+ * Represents a set of memory ranges
+ */
+struct VmaRangeGroup {
+ std::vector<VmaRange> ranges;
+
+ /**
+ * Compute intersection coverage between |range| and |this->ranges|
+ * and append it to |out_memres|
+ */
+ void compute_coverage(const VmaRange& range, VmaRangeGroup& out_memres) const;
+
+ /**
+ * Apply an offset to all existing |ranges|.
+ */
+ void apply_offset(uint64_t offset);
+
+ /**
+ * Computes total resident bytes from existing set of memory ranges.
+ */
+ uint64_t compute_total_size();
+};
+
+/**
+ * Represents useful immutable metadata for zip entry
+ */
+struct ZipEntryInfo {
+ std::string name;
+ uint64_t offset_in_zip;
+ uint64_t file_size_bytes;
+ uint64_t uncompressed_size;
+};
+
+/**
+ * Represents the resident memory coverage for a zip entry within a zip file.
+ */
+struct ZipEntryCoverage {
+ ZipEntryInfo info;
+
+ /**
+ * Contains all the coverage ranges if any have been computed with |compute_coverage|
+ * and their offsets will be the absolute global offset from the zip file start.
+ */
+ VmaRangeGroup coverage;
+
+ /**
+ * Computes the intersection coverage for the current zip file entry
+ * resident memory against a provided |probe| representing another set
+ * of ranges.
+ */
+ ZipEntryCoverage compute_coverage(const VmaRangeGroup& probe) const;
+};
+
+// Class used for inspecting resident memory for entries within a zip file
+class ZipMemInspector {
+ /**
+ * Stored probe of resident ranges either computed or provided by user.
+ */
+ VmaRangeGroup* probe_resident_ = nullptr;
+
+ /**
+ * List of file entries within zip file.
+ */
+ std::vector<ZipEntryInfo> entry_infos_;
+
+ /**
+ * Path to zip file.
+ */
+ std::string filename_;
+
+ /**
+ * Result of computing coverage operations.
+ */
+ std::vector<ZipEntryCoverage> entry_coverages_;
+
+ /**
+ * Handle that allows reading the zip entries.
+ */
+ ZipArchiveHandle handle_;
+
+ public:
+ ZipMemInspector(std::string filename) : filename_(filename) {}
+ ~ZipMemInspector();
+
+ /**
+ * Reads zip file and computes resident memory coverage per zip entry if
+ * a probe is provided, if no probe is provided, then whole file coverage
+ * will be assumed.
+ *
+ * Note: If any zip entries have been manually added via |add_file_info|
+ * then coverage will be only computed against manually added entries.
+ *
+ * @return 0 on success and 1 on error
+ */
+ int compute_per_file_coverage();
+
+ /**
+ * Computes resident memory for the entire zip file.
+ *
+ * @return 0 on success, 1 on failure
+ */
+ int probe_resident();
+
+ /**
+ * Retrieves the currently set probe if any exists.
+ */
+ VmaRangeGroup* get_probe();
+
+ /**
+ * Sets probe data in case you decide to pass a previously taken probe instead of a live taken
+ * one.
+ */
+ void set_existing_probe(VmaRangeGroup* probe);
+
+ /**
+ * Returns the result of memory coverage of each file if any has been computed via
+ * |compute_per_file_coverage|.
+ */
+ std::vector<ZipEntryCoverage>& get_file_coverages();
+
+ /**
+ * Returns the file information for each zip entry.
+ */
+ std::vector<ZipEntryInfo>& get_file_infos();
+
+ /**
+ * Add a zip entry manually.
+ *
+ * Note: Zip entries are usually retrieved by reading the |filename_| so
+ * this method is mostly used for cases where client wants control of
+ * zip file reading or for testing.
+ */
+ void add_file_info(ZipEntryInfo& file);
+
+ /**
+ * Computes the intersection coverage between provided |files| and |probe|.
+ *
+ * @return result of coverage computation
+ */
+ static std::vector<ZipEntryCoverage> compute_coverage(
+ const std::vector<ZipEntryCoverage>& files, VmaRangeGroup* probe);
+
+ private:
+ /**
+ * Read files and zip relative offsets for them.
+ *
+ * @return 0 on success, 1 on failure.
+ */
+ int read_files_and_offsets();
+};
+
+/**
+ * Retrieve file size in bytes for |file|
+ *
+ * @return positive value with file size on success, otherwise, returns -1 on error.
+ */
+int64_t get_file_size(const std::string& file);
+
+/**
+ * @brief Probe resident memory for a currently opened file in the system.
+ *
+ * @param probed_file File to probe as defined by its path.
+ * @param out_resident_mem Inspection result. This is populated when called.
+ * @param pages_per_mincore Size of mincore window used, bigger means more memory used
+ * during operation but slightly faster.
+ * @return 0 on success or on failure a non-zero error code from the following list:
+ * MEMINSPECT_FAIL_OPEN, MEMINSPECT_FAIL_FSTAT, MEMINSPECT_FAIL_MINCORE
+ */
+int probe_resident_memory(std::string probed_file, VmaRangeGroup& out_resident_mem,
+ int pages_per_mincore = DEFAULT_PAGES_PER_MINCORE);
+
+/**
+ * @brief Align vma ranges to a certain page size
+ *
+ * @param ranges vma ranges that have to be aligned
+ * @param alignment Desired alignment, this is usually the page size.
+ */
+void align_ranges(std::vector<VmaRange>& ranges, unsigned int alignment);
+
+/**
+ * @brief Merges a list of ranges following a union-like merge which
+ * means that two ranges that overlap will avoid double accounting for
+ * overlaps.
+ *
+ * @param ranges vma ranges that need to be merged.
+ * @return new vector with ranges merged.
+ */
+std::vector<VmaRange> merge_ranges(const std::vector<VmaRange>& ranges); \ No newline at end of file
diff --git a/pinner/include/pin_utils.h b/pinner/include/pin_utils.h
new file mode 100644
index 00000000..b6066d63
--- /dev/null
+++ b/pinner/include/pin_utils.h
@@ -0,0 +1,116 @@
+#pragma once
+
+#include <list>
+#include "meminspect.h"
+
+struct PinConfigFile {
+ std::string filename;
+
+ // File relative offsets
+ std::vector<VmaRange> ranges;
+
+ ZipEntryCoverage to_zipfilemem(const ZipEntryInfo& info);
+};
+
+struct PinConfig {
+ std::list<PinConfigFile> files_;
+
+ int parse(std::string filename, bool verbose = false);
+};
+
+/**
+ * @brief Generate a pinlist file from a given list of vmas containing a list of 4-byte pairs
+ * representing (4-byte offset, 4-byte len) contiguous in memory and they are stored in big endian
+ * format.
+ *
+ * @param output_file Output file to write pinlist
+ * @param vmas_to_pin Set of vmas to write into pinlist file.
+ * @param write_quota Specifies a maximum amount o bytes to be written to the pinlist file
+ * or -1 means no limit.
+ * @return 0 on success, non-zero on failure
+ */
+int write_pinlist_file(const std::string& output_file, const std::vector<VmaRange>& vmas_to_pin,
+ int64_t write_quota = -1);
+
+/**
+ * @brief This method is the counter part of @see write_pinlist_file(). It will read an existing
+ * pinlist file.
+ *
+ * @param pinner_file Input pinlist file
+ * @param pinranges Vmas read from pinlist file. This is populated on call.
+ * @return 0 on success, non-zero on failure
+ */
+int read_pinlist_file(const std::string& pinner_file, /*out*/ std::vector<VmaRange>& pinranges);
+
+enum ProbeType {
+ UNSET, // No probe setup
+ GENERATE, // Generate a probe
+ CUSTOM // User generated probe
+};
+
+class PinTool {
+ public:
+ enum DumpType { PROBE, FILE_COVERAGE, FILTERED };
+
+ private:
+ std::string input_file_;
+ std::string custom_probe_file_;
+ PinConfig* pinconfig_;
+ std::vector<ZipEntryCoverage> filtered_files_;
+ bool verbose_;
+ ZipMemInspector* zip_inspector_ = nullptr;
+
+ public:
+ PinTool(const std::string& input_file) : input_file_(input_file) {
+ zip_inspector_ = new ZipMemInspector(input_file_);
+ }
+
+ ~PinTool() {
+ delete zip_inspector_;
+ delete pinconfig_;
+ }
+
+ void set_verbose_output(bool verbose);
+
+ // Read |probe_file| which should be a pinlist.meta style
+ // file and use it as current probe.
+ void read_probe_from_pinlist(std::string probe_file);
+
+ // Compute a resident memory probe for |input_file_|
+ int probe_resident();
+
+ // Compute coverage for each zip entry contained within
+ // |input_file_|.
+ // Note: It only works for zip files
+ void compute_zip_entry_coverages();
+
+ /**
+ * Filter coverages based on a provided pinconfig style file
+ * See README.md for sample structure of pinconfig file.
+ *
+ * Note: It only works for zip files, for non zip files, this will be
+ * a no-op.
+ */
+ void filter_zip_entry_coverages(const std::string& pinconfig_file);
+
+ void filter_zip_entry_coverages(PinConfig* pinconfig);
+
+ /**
+ * Dumps output of existing coverages to console for |type|.
+ */
+ void dump_coverages(DumpType type);
+
+ /**
+ * Writes coverages into a pinlist.meta style file.
+ *
+ * @param write_quota Maximum bytes allowed to be written to file.
+ */
+ void write_coverages_as_pinlist(std::string output_pinlist, int64_t write_quota = -1);
+
+ std::vector<ZipEntryCoverage> get_filtered_zip_entries();
+
+ /**
+ * Sets a user defined inspector, currently only used for testing.
+ */
+ void set_custom_zip_inspector(ZipMemInspector* inspector);
+}; \ No newline at end of file
diff --git a/pinner/meminspect.cpp b/pinner/meminspect.cpp
new file mode 100644
index 00000000..84e00928
--- /dev/null
+++ b/pinner/meminspect.cpp
@@ -0,0 +1,324 @@
+#include "meminspect.h"
+#include <android-base/unique_fd.h>
+#include "ziparchive/zip_archive.h"
+
+using namespace std;
+using namespace android::base;
+using namespace ::android::base;
+
+const static VmaRange VMA_RANGE_EMPTY = VmaRange(0, 0);
+
+uint32_t VmaRange::end_offset() const {
+ return offset + length;
+}
+
+uint64_t VmaRangeGroup::compute_total_size() {
+ uint64_t total_size = 0;
+ for (auto&& range : ranges) {
+ total_size += range.length;
+ }
+ return total_size;
+}
+
+void VmaRangeGroup::apply_offset(uint64_t offset) {
+ for (auto&& range : ranges) {
+ range.offset += offset;
+ }
+}
+
+void VmaRangeGroup::compute_coverage(const VmaRange& range, VmaRangeGroup& out_memres) const {
+ for (auto&& resident_range : ranges) {
+ VmaRange intersect_res = resident_range.intersect(range);
+ if (!intersect_res.is_empty()) {
+ out_memres.ranges.push_back(intersect_res);
+ }
+ }
+}
+
+bool VmaRange::is_empty() const {
+ return length == 0;
+}
+
+VmaRange VmaRange::intersect(const VmaRange& target) const {
+ // First check if the slice is outside our range
+ if (target.end_offset() <= this->offset) {
+ return VMA_RANGE_EMPTY;
+ }
+ if (target.offset >= this->end_offset()) {
+ return VMA_RANGE_EMPTY;
+ }
+ VmaRange result;
+ // the slice should now be inside the range so compute the intersection.
+ result.offset = std::max(target.offset, this->offset);
+ uint32_t res_end = std::min(target.end_offset(), end_offset());
+ result.length = res_end - result.offset;
+
+ return result;
+}
+
+VmaRange VmaRange::union_merge(const VmaRange& target) const {
+ VmaRange result = intersect(target);
+ if (result.is_empty()) {
+ // Disjointed ranges, no merge.
+ return VMA_RANGE_EMPTY;
+ }
+
+ // Since there is an intersection, merge ranges between lowest
+ // and highest value.
+ result.offset = std::min(offset, target.offset);
+ uint32_t res_end = std::max(target.end_offset(), end_offset());
+ result.length = res_end - result.offset;
+ return result;
+}
+
+void align_ranges(std::vector<VmaRange>& vmas_to_align, unsigned int alignment) {
+ for (auto&& vma_to_align : vmas_to_align) {
+ uint32_t unaligned_offset = vma_to_align.offset % alignment;
+ vma_to_align.offset -= unaligned_offset;
+ vma_to_align.length += unaligned_offset;
+ }
+}
+
+bool compare_range(VmaRange& a, VmaRange& b) {
+ return a.offset < b.offset;
+}
+
+std::vector<VmaRange> merge_ranges(const std::vector<VmaRange>& ranges) {
+ if (ranges.size() <= 1) {
+ // Not enough ranges to perform a merge.
+ return ranges;
+ }
+
+ std::vector<VmaRange> to_merge_ranges = ranges;
+ std::vector<VmaRange> merged_ranges;
+ // Sort the ranges to make a slightly more efficient merging.
+ std::sort(to_merge_ranges.begin(), to_merge_ranges.end(), compare_range);
+
+ // The first element will always start as-is, then start merging with subsequent elements.
+ merged_ranges.push_back(to_merge_ranges[0]);
+ for (int iMerged = 0, iTarget = 1; iTarget < to_merge_ranges.size(); ++iTarget) {
+ VmaRange merged = merged_ranges[iMerged].union_merge(to_merge_ranges[iTarget]);
+ if (!merged.is_empty()) {
+ // Merge was successful, swallow range.
+ merged_ranges[iMerged] = merged;
+ } else {
+ // Merge failed, add disjointed range.
+ merged_ranges.push_back(to_merge_ranges[iTarget]);
+ ++iMerged;
+ }
+ }
+
+ return merged_ranges;
+}
+
+int64_t get_file_size(const std::string& file) {
+ unique_fd file_ufd(open(file.c_str(), O_RDONLY));
+ int fd = file_ufd.get();
+ if (fd == -1) {
+ return -1;
+ }
+
+ struct stat fstat_res;
+ int res = fstat(fd, &fstat_res);
+ if (res == -1) {
+ return -1;
+ }
+
+ return fstat_res.st_size;
+}
+
+int probe_resident_memory(string probed_file,
+ /*out*/ VmaRangeGroup& resident_ranges, int pages_per_mincore) {
+ unique_fd probed_file_ufd(open(probed_file.c_str(), O_RDONLY));
+ int probe_fd = probed_file_ufd.get();
+ if (probe_fd == -1) {
+ return MEMINSPECT_FAIL_OPEN;
+ }
+
+ int64_t total_bytes = get_file_size(probed_file);
+ if (total_bytes < 0) {
+ return MEMINSPECT_FAIL_FSTAT;
+ }
+
+ char* base_address =
+ (char*)mmap(0, (uint64_t)total_bytes, PROT_READ, MAP_SHARED, probe_fd, /*offset*/ 0);
+
+ // this determines how many pages to inspect per mincore syscall
+ unsigned char* window = new unsigned char[pages_per_mincore];
+
+ unsigned int page_size = sysconf(_SC_PAGESIZE);
+ unsigned long bytes_inspected = 0;
+
+ // total bytes in inspection window
+ unsigned long window_bytes = page_size * pages_per_mincore;
+
+ char* window_base;
+ bool started_vma_range = false;
+ uint32_t resident_vma_start_offset = 0;
+ for (window_base = base_address; bytes_inspected < total_bytes;
+ window_base += window_bytes, bytes_inspected += window_bytes) {
+ int res = mincore(window_base, window_bytes, window);
+ if (res != 0) {
+ if (errno == ENOMEM) {
+ // Did not find page, maybe it's a hole.
+ continue;
+ }
+ return MEMINSPECT_FAIL_MINCORE;
+ }
+ // Inspect the provided mincore window result sequentially
+ // and as soon as a change in residency happens a range is
+ // created or finished.
+ for (int iWin = 0; iWin < pages_per_mincore; ++iWin) {
+ if ((window[iWin] & (unsigned char)1) != 0) {
+ // Page is resident
+ if (!started_vma_range) {
+ // End of range
+ started_vma_range = true;
+ uint32_t window_offset = iWin * page_size;
+ resident_vma_start_offset = window_base + window_offset - base_address;
+ }
+ } else {
+ // Page is not resident
+ if (started_vma_range) {
+ // Start of range
+ started_vma_range = false;
+ uint32_t window_offset = iWin * page_size;
+ uint32_t resident_vma_end_offset = window_base + window_offset - base_address;
+ uint32_t resident_len = resident_vma_end_offset - resident_vma_start_offset;
+ VmaRange vma_range(resident_vma_start_offset, resident_len);
+ resident_ranges.ranges.push_back(vma_range);
+ }
+ }
+ }
+ }
+ // This was the last window, so close any opened vma range
+ if (started_vma_range) {
+ started_vma_range = false;
+ uint32_t in_memory_vma_end = window_base - base_address;
+ uint32_t resident_len = in_memory_vma_end - resident_vma_start_offset;
+ VmaRange vma_range(resident_vma_start_offset, resident_len);
+ resident_ranges.ranges.push_back(vma_range);
+ }
+
+ return 0;
+}
+
+ZipMemInspector::~ZipMemInspector() {
+ CloseArchive(handle_);
+ delete probe_resident_;
+}
+
+ZipEntryCoverage ZipEntryCoverage::compute_coverage(const VmaRangeGroup& probe) const {
+ ZipEntryCoverage file_coverage;
+ file_coverage.info = info;
+
+ // Compute coverage for each range in file against probe which represents a set of ranges.
+ for (auto&& range : coverage.ranges) {
+ probe.compute_coverage(range, file_coverage.coverage);
+ }
+
+ return file_coverage;
+}
+
+std::vector<ZipEntryCoverage> ZipMemInspector::compute_coverage(
+ const std::vector<ZipEntryCoverage>& files, VmaRangeGroup* probe) {
+ if (probe == nullptr) {
+ // No probe to calculate coverage against, so coverage is zero.
+ return std::vector<ZipEntryCoverage>();
+ }
+
+ std::vector<ZipEntryCoverage> file_coverages;
+ // Find the file coverage against provided probe.
+ for (auto&& file : files) {
+ // For each file, compute coverage against the probe which represents a list of ranges.
+ ZipEntryCoverage file_coverage = file.compute_coverage(*probe);
+ file_coverages.push_back(file_coverage);
+ }
+
+ return file_coverages;
+}
+
+void ZipMemInspector::add_file_info(ZipEntryInfo& file) {
+ entry_infos_.push_back(file);
+}
+
+int ZipMemInspector::compute_per_file_coverage() {
+ if (entry_infos_.empty()) {
+ // We haven't read the file information yet, so do it now.
+ if (read_files_and_offsets()) {
+ cerr << "Could not read zip entries to compute coverages." << endl;
+ return 1;
+ }
+ }
+
+ // All existing files should consider their whole memory as present by default.
+ std::vector<ZipEntryCoverage> entry_coverages;
+ for (auto&& entry_info : entry_infos_) {
+ ZipEntryCoverage entry_coverage;
+ entry_coverage.info = entry_info;
+ VmaRange file_vma_range(entry_info.offset_in_zip, entry_info.file_size_bytes);
+ entry_coverage.coverage.ranges.push_back(file_vma_range);
+ entry_coverage.coverage.compute_total_size();
+ entry_coverages.push_back(entry_coverage);
+ }
+
+ if (probe_resident_ != nullptr) {
+ // We decided to compute coverage based on a probe
+ entry_coverages_ = compute_coverage(entry_coverages, probe_resident_);
+ } else {
+ // No probe means whole file coverage
+ entry_coverages_ = entry_coverages;
+ }
+
+ return 0;
+}
+
+VmaRangeGroup* ZipMemInspector::get_probe() {
+ return probe_resident_;
+}
+
+void ZipMemInspector::set_existing_probe(VmaRangeGroup* probe) {
+ this->probe_resident_ = probe;
+}
+
+std::vector<ZipEntryCoverage>& ZipMemInspector::get_file_coverages() {
+ return entry_coverages_;
+}
+
+int ZipMemInspector::probe_resident() {
+ probe_resident_ = new VmaRangeGroup();
+ int res = probe_resident_memory(filename_, *probe_resident_);
+ if (res != 0) {
+ // Failed to probe
+ return res;
+ }
+
+ return 0;
+}
+
+std::vector<ZipEntryInfo>& ZipMemInspector::get_file_infos() {
+ return entry_infos_;
+}
+
+int ZipMemInspector::read_files_and_offsets() {
+ if (OpenArchive(filename_.c_str(), &handle_) < 0) {
+ return 1;
+ }
+ void* cookie;
+ int res = StartIteration(handle_, &cookie);
+ if (res != 0) {
+ return 1;
+ }
+
+ ZipEntry64 entry;
+ string name;
+ while (Next(cookie, &entry, &name) == 0) {
+ ZipEntryInfo file;
+ file.name = name;
+ file.offset_in_zip = entry.offset;
+ file.file_size_bytes = entry.compressed_length;
+ file.uncompressed_size = entry.uncompressed_length;
+ entry_infos_.push_back(file);
+ }
+ return 0;
+}
diff --git a/pinner/pin_utils.cpp b/pinner/pin_utils.cpp
new file mode 100644
index 00000000..7c746bb0
--- /dev/null
+++ b/pinner/pin_utils.cpp
@@ -0,0 +1,347 @@
+#include "pin_utils.h"
+#include <android-base/parseint.h>
+#include <algorithm>
+#include <fstream>
+#include <map>
+#include <string_view>
+#include <utility>
+#include <vector>
+
+using namespace std;
+using namespace android::base;
+
+int write_pinlist_file(const std::string& output_file,
+ const std::vector<ZipEntryCoverage>& files_to_write, int64_t write_quota) {
+ std::vector<VmaRange> ranges;
+ for (auto&& file : files_to_write) {
+ ranges.insert(ranges.end(), file.coverage.ranges.begin(), file.coverage.ranges.end());
+ }
+ return write_pinlist_file(output_file, ranges, write_quota);
+}
+
+int write_pinlist_file(const std::string& output_file, const std::vector<VmaRange>& vmas_to_write,
+ int64_t write_quota) {
+ ofstream pinlist_file(output_file);
+ if (pinlist_file.fail()) {
+ return 1;
+ }
+ int64_t total_written = 0;
+ unsigned int page_size = sysconf(_SC_PAGESIZE);
+ const bool has_quota = write_quota > 0;
+ bool reached_quota = false;
+
+ // The PinnerService does not require aligned offsets, however, aligning
+ // allows our summary results to be accurate and avoids over-accounting
+ // of pinning in PinnerService.
+ std::vector<VmaRange> processed_vmas_to_write = vmas_to_write;
+ align_ranges(processed_vmas_to_write, page_size);
+
+ // When we page-align the ranges, we may cause overlaps between ranges
+ // as we elongate the begin offset to match the page the previous
+ // range may end up overlapping the current one.
+ processed_vmas_to_write = merge_ranges(processed_vmas_to_write);
+
+ for (auto&& processed_vma_to_write : processed_vmas_to_write) {
+ uint32_t vma_start_offset = processed_vma_to_write.offset;
+ uint32_t vma_length = processed_vma_to_write.length;
+ if (has_quota && (total_written + vma_length > write_quota)) {
+ // We would go beyond quota, set the maximum allowed write and exit.
+ vma_length = write_quota - total_written;
+ reached_quota = true;
+ }
+ // Transform to BigEndian as PinnerService requires that endianness for reading.
+ uint32_t vma_start_offset_be = htobe32(vma_start_offset);
+ uint32_t vma_length_be = htobe32(vma_length);
+ cout << "Pinlist Writing start=" << vma_start_offset << " bytes=" << vma_length << endl;
+ pinlist_file.write(reinterpret_cast<char*>(&vma_start_offset_be),
+ sizeof(vma_start_offset_be));
+ if (pinlist_file.fail()) {
+ return 1;
+ }
+ pinlist_file.write(reinterpret_cast<char*>(&vma_length_be), sizeof(vma_length_be));
+ total_written += vma_length;
+ if (pinlist_file.fail()) {
+ return 1;
+ }
+
+ if (reached_quota) {
+ break;
+ }
+ }
+ return 0;
+}
+
+int read_pinlist_file(const std::string& pinner_file, /*out*/ std::vector<VmaRange>& pinranges) {
+ ifstream pinlist_file(pinner_file);
+ if (pinlist_file.fail()) {
+ return 1;
+ }
+
+ uint32_t vma_start;
+ uint32_t vma_length;
+ while (!pinlist_file.eof()) {
+ pinlist_file.read(reinterpret_cast<char*>(&vma_start), sizeof(vma_start));
+ pinlist_file.read(reinterpret_cast<char*>(&vma_length), sizeof(vma_length));
+ if (pinlist_file.fail()) {
+ return 1;
+ }
+ vma_start = betoh32(vma_start);
+ vma_length = betoh32(vma_length);
+ pinranges.push_back(VmaRange(vma_start, vma_length));
+ }
+
+ return 0;
+}
+
+ZipEntryCoverage PinConfigFile::to_zipfilemem(const ZipEntryInfo& info) {
+ ZipEntryCoverage file;
+ file.info = info;
+
+ if (ranges.empty()) {
+ cout << "No ranges found for file " << info.name << " creating entire file range" << endl;
+ // Any file coming from pinconfig without explicit
+ // ranges will be assumed to be wanted in its entirety
+ ranges.push_back(VmaRange(0, info.file_size_bytes));
+ }
+
+ file.coverage.ranges = ranges;
+
+ // Offsets specified in pinconfig file are relative to the file
+ // so transform to zip global offsets which are used for coverage
+ // computations.
+ file.coverage.apply_offset(info.offset_in_zip);
+
+ file.coverage.compute_total_size();
+ return file;
+}
+
+int PinConfig::parse(std::string config_file, bool verbose) {
+ ifstream file(config_file);
+ string file_in_zip;
+ if (verbose) {
+ cout << "Parsing file: " << config_file << endl;
+ }
+ string token;
+ file >> token;
+ while (!file.eof()) {
+ if (token == "file") {
+ file >> file_in_zip;
+ PinConfigFile pin_config_file;
+ pin_config_file.filename = file_in_zip;
+ file >> token;
+ while (token != "file" && !file.eof()) {
+ VmaRange range;
+ // Inner parsing loop for per file config.
+ if (token == "offset") {
+ file >> token;
+ android::base::ParseUint(token, &range.offset);
+ file >> token;
+ if (token != "len") {
+ cerr << "Malformed file, expected 'len' after offset" << endl;
+ return 1;
+ }
+ file >> token;
+ android::base::ParseUint(token, &range.length);
+ pin_config_file.ranges.push_back(range);
+ }
+ file >> token;
+ }
+ files_.push_back(pin_config_file);
+ } else {
+ cerr << "Unexpected token: " << token << ". Exit read" << endl;
+ return 1;
+ }
+ }
+
+ if (files_.empty()) {
+ cerr << "Failed parsing pinconfig file, no entries found." << endl;
+ return 1;
+ }
+
+ if (verbose) {
+ cout << "Finished parsing Pinconfig file" << endl;
+ for (auto&& pin_file : files_) {
+ cout << "file=" << pin_file.filename << endl;
+ for (auto&& range : pin_file.ranges) {
+ cout << "offset=" << range.offset << " bytes=" << range.length << endl;
+ }
+ }
+ }
+
+ return 0;
+}
+
+void PinTool::set_custom_zip_inspector(ZipMemInspector* inspector) {
+ delete zip_inspector_;
+ zip_inspector_ = inspector;
+}
+
+void PinTool::set_verbose_output(bool verbose) {
+ verbose_ = verbose;
+}
+
+void PinTool::read_probe_from_pinlist(std::string custom_probe_file) {
+ custom_probe_file_ = custom_probe_file;
+ VmaRangeGroup* custom_probe = new VmaRangeGroup();
+ read_pinlist_file(custom_probe_file_, custom_probe->ranges);
+ custom_probe->compute_total_size();
+ if (custom_probe->ranges.empty()) {
+ cerr << "Did not find any memory range in " << custom_probe_file_ << endl;
+ delete custom_probe;
+ return;
+ }
+ zip_inspector_->set_existing_probe(custom_probe);
+}
+
+int PinTool::probe_resident() {
+ return zip_inspector_->probe_resident();
+}
+
+void PinTool::compute_zip_entry_coverages() {
+ zip_inspector_->compute_per_file_coverage();
+ if (verbose_) {
+ std::vector<ZipEntryInfo> files = zip_inspector_->get_file_infos();
+ for (auto&& file : files) {
+ cout << "file found. name=" << file.name << " offset=" << file.offset_in_zip
+ << " uncompressed=" << file.uncompressed_size
+ << " compressed=" << file.file_size_bytes << endl
+ << endl;
+ }
+ }
+}
+
+void PinTool::dump_coverages(PinTool::DumpType dump_type) {
+ std::vector<ZipEntryCoverage>* file_coverages;
+ if (dump_type == PinTool::DumpType::FILTERED) {
+ file_coverages = &filtered_files_;
+ } else if (dump_type == PinTool::DumpType::FILE_COVERAGE) {
+ file_coverages = &(zip_inspector_->get_file_coverages());
+ } else { // PinTool::DumpType::PROBE
+ VmaRangeGroup* probe = zip_inspector_->get_probe();
+ file_coverages = new vector<ZipEntryCoverage>();
+ ZipEntryCoverage file;
+ file.coverage = *probe;
+ file.info.name = input_file_;
+ file.info.offset_in_zip = 0;
+ uint64_t file_size_bytes = get_file_size(input_file_);
+ if (file_size_bytes == -1) {
+ cerr << "Failed to dump, cannot fstat file: " << input_file_ << endl;
+ delete file_coverages;
+ return;
+ }
+ file.info.file_size_bytes = file_size_bytes;
+ file_coverages->push_back(file);
+ }
+
+ for (auto&& file : *file_coverages) {
+ uint64_t total_size = file.coverage.compute_total_size();
+ cout << file.info.name << " size(B)=" << file.info.file_size_bytes
+ << " resident(B)=" << total_size
+ << " resident(%)=" << (double)(total_size) / file.info.file_size_bytes * 100.0 << endl;
+ if (verbose_) {
+ cout << "file_base_zip_offset=" << file.info.offset_in_zip << endl;
+ }
+ cout << "file resident ranges" << endl;
+ if (dump_type != DumpType::PROBE) {
+ for (auto&& range : file.coverage.ranges) {
+ // The offset in the range represents the absolute absolute offset relative to the
+ // zip so substract the file base offset to get the relative offset within the file
+ // which may be what is worth for a user to specify in pinconfig.txt files.
+ uint64_t offset_in_file = range.offset - file.info.offset_in_zip;
+
+ cout << "zip_offset=" << range.offset << " file_offset=" << offset_in_file
+ << " total_bytes=" << range.length << endl;
+ }
+ } else {
+ for (auto&& range : file.coverage.ranges) {
+ cout << "file_offset=" << range.offset << " total_bytes=" << range.length << endl;
+ }
+ }
+ cout << endl;
+ }
+ cout << endl;
+ if (dump_type == DumpType::PROBE) {
+ // For other dump types we do not create memory, we reuse from class.
+ delete file_coverages;
+ }
+}
+
+void PinTool::filter_zip_entry_coverages(const std::string& pinconfig_filename) {
+ if (pinconfig_filename.length() == 0) {
+ // Nothing to do.
+ return;
+ }
+
+ PinConfig* pinconfig = new PinConfig();
+ if (pinconfig->parse(pinconfig_filename, verbose_) > 0) {
+ cerr << "Failed parsing pinconfig file " << pinconfig_filename << ". Skip filtering";
+ delete pinconfig;
+ return;
+ }
+
+ filter_zip_entry_coverages(pinconfig);
+}
+
+void PinTool::filter_zip_entry_coverages(PinConfig* pinconfig) {
+ pinconfig_ = pinconfig;
+
+ // Filter based on the per file configuration.
+ vector<ZipEntryCoverage> file_coverages = zip_inspector_->get_file_coverages();
+ vector<ZipEntryCoverage>& filtered_files = filtered_files_;
+
+ for (auto&& file_coverage : file_coverages) {
+ for (auto&& pinconfig_file : pinconfig_->files_) {
+ // Match each zip entry against every pattern in filter file.
+ std::string_view file_coverage_view(file_coverage.info.name.c_str());
+ std::string_view pinconfig_view(pinconfig_file.filename.c_str());
+ if (file_coverage_view.find(pinconfig_view) != std::string_view::npos) {
+ // Now that we found a match, create a file with offsets that are global to zip file
+ ZipEntryCoverage file_in_config = pinconfig_file.to_zipfilemem(file_coverage.info);
+ if (verbose_) {
+ cout << "Found a match: file=" << file_coverage.info.name
+ << " matching filter=" << pinconfig_file.filename << endl;
+ for (auto&& range : file_in_config.coverage.ranges) {
+ cout << "zip_offset=" << range.offset << " bytes=" << range.length << endl;
+ }
+ }
+ ZipEntryCoverage filtered_file =
+ file_coverage.compute_coverage(file_in_config.coverage);
+ filtered_files.push_back(filtered_file);
+ break;
+ }
+ }
+ }
+}
+
+std::vector<ZipEntryCoverage> PinTool::get_filtered_zip_entries() {
+ return filtered_files_;
+}
+
+void PinTool::write_coverages_as_pinlist(std::string output_pinlist, int64_t write_quota) {
+ std::vector<ZipEntryCoverage>* pinlist_coverages = nullptr;
+ if (!filtered_files_.empty()) {
+ // Highest preference is writing filtered files if they exist
+ if (verbose_) {
+ cout << "Writing pinconfig filtered file coverages" << endl;
+ }
+ pinlist_coverages = &filtered_files_;
+ } else if (!zip_inspector_->get_file_coverages().empty()) {
+ // Fallback to looking for file coverage computation
+ pinlist_coverages = &zip_inspector_->get_file_coverages();
+ if (verbose_) {
+ cout << "Writing regular file coverages." << endl;
+ }
+ }
+ if (pinlist_coverages == nullptr) {
+ cerr << "Failed to find coverage to write to: " << output_pinlist << endl;
+ return;
+ }
+ int res = write_pinlist_file(output_pinlist, *pinlist_coverages, write_quota);
+ if (res > 0) {
+ cerr << "Failed to write pin file at: " << output_pinlist << endl;
+ } else {
+ if (verbose_) {
+ cout << "Finished writing pin file at: " << output_pinlist << endl;
+ }
+ }
+} \ No newline at end of file
diff --git a/pinner/pintool.cpp b/pinner/pintool.cpp
new file mode 100644
index 00000000..9aa47b7b
--- /dev/null
+++ b/pinner/pintool.cpp
@@ -0,0 +1,343 @@
+#include <android-base/parseint.h>
+#include <fcntl.h>
+#include <sys/endian.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <iostream>
+#include <string>
+#include <vector>
+
+#include <meminspect.h>
+#include <pin_utils.h>
+
+using namespace std;
+using namespace android::base;
+
+enum ToolMode {
+ MAPPED_FILE, // Files that are mapped in memory
+ PINLIST, // pinlist.meta style file
+ UNKNOWN
+};
+
+void print_pinlist_ranges(const std::vector<VmaRange>& ranges) {
+ cout << "--pinlist memory ranges--" << endl;
+ for (auto&& range : ranges) {
+ cout << "start=" << range.offset << " bytes=" << range.length << endl;
+ }
+}
+
+void print_pinlist_summary(const std::vector<VmaRange>& ranges) {
+ cout << "--pinlist summary--" << endl;
+ uint64_t total_bytes = 0;
+ for (auto&& range : ranges) {
+ total_bytes += range.length;
+ }
+ cout << "total_bytes_to_pin=" << total_bytes << endl;
+}
+
+int perform_file_action(const vector<string>& options) {
+ std::string custom_probe_file;
+ std::string output_file;
+ std::string pinconfig_file;
+
+ bool verbose = false;
+ bool is_zip = false;
+ bool dump_results = false;
+ ProbeType probe_type = UNSET;
+ int64_t write_quota = -1; // unbounded by default
+
+ if (options.empty()) {
+ cerr << "Missing filename for file action, see usage for details." << endl;
+ return 1;
+ }
+
+ std::string input_file = options[0];
+
+ // Validate that the file exists.
+ if (get_file_size(input_file) == -1) {
+ cerr << "Error: Could not read file: " << input_file << endl;
+ return 1;
+ }
+
+ if (input_file.empty()) {
+ cerr << "Error: Should specify an input file." << endl;
+ return 1;
+ }
+
+ // Parse flags
+ for (int i = 1; i < options.size(); ++i) {
+ string option = options[i];
+ if (option == "--gen-probe") {
+ if (probe_type != ProbeType::UNSET) {
+ cerr << "Should only specify one probe treatment. See usage for details." << endl;
+ return 1;
+ }
+ probe_type = ProbeType::GENERATE;
+ continue;
+ }
+
+ if (option == "--use-probe") {
+ if (probe_type != ProbeType::UNSET) {
+ cerr << "Should only specify one probe treatment. See usage for details." << endl;
+ return 1;
+ }
+ probe_type = ProbeType::CUSTOM;
+ ++i;
+ custom_probe_file = options[i];
+ continue;
+ }
+ if (option == "--pinconfig") {
+ ++i;
+ pinconfig_file = options[i];
+ continue;
+ }
+ if (option == "-o") {
+ ++i;
+ output_file = options[i];
+ continue;
+ }
+ if (option == "--quota") {
+ ++i;
+ android::base::ParseInt(options[i], &write_quota);
+ continue;
+ }
+ if (option == "-v") {
+ verbose = true;
+ continue;
+ }
+ if (option == "--zip") {
+ is_zip = true;
+ continue;
+ }
+ if (option == "--dump") {
+ dump_results = true;
+ continue;
+ }
+ }
+
+ if (verbose) {
+ cout << "Setting output pinlist file: " << output_file.c_str() << endl;
+ cout << "Setting input file: " << input_file.c_str() << endl;
+ cout << "Setting pinconfig file: " << pinconfig_file.c_str() << endl;
+ cout << "Setting custom probe file: " << custom_probe_file.c_str() << endl;
+ cout << "Setting probe type: " << probe_type << endl;
+ cout << "Dump enabled: " << dump_results << endl;
+ cout << "Is Zip file: " << is_zip << endl;
+ if (write_quota != -1) {
+ cout << "Set Write quota: " << write_quota << endl;
+ }
+ }
+
+ PinTool pintool(input_file);
+
+ if (is_zip) {
+ pintool.set_verbose_output(verbose);
+ if (probe_type == ProbeType::CUSTOM) {
+ if (verbose) {
+ cout << "Using custom probe file: " << custom_probe_file << endl;
+ }
+ pintool.read_probe_from_pinlist(custom_probe_file);
+ } else if (probe_type == ProbeType::GENERATE) {
+ if (verbose) {
+ cout << "Generating probe" << endl;
+ }
+ int res = pintool.probe_resident();
+ if (res > 0) {
+ cerr << "Failed to generate probe. Error Code: " << res << endl;
+ return 1;
+ }
+ }
+ pintool.compute_zip_entry_coverages();
+
+ if (pinconfig_file.length() > 0) {
+ // We have provided a pinconfig file so perform filtering
+ // of computed coverages based on it.
+ pintool.filter_zip_entry_coverages(pinconfig_file);
+ }
+
+ if (dump_results) {
+ cout << endl << "----Unfiltered file coverages----" << endl << endl;
+ pintool.dump_coverages(PinTool::DumpType::FILE_COVERAGE);
+
+ if (pinconfig_file.length() > 0) {
+ cout << endl << "----Filtered file coverages----" << endl << endl;
+ pintool.dump_coverages(PinTool::DumpType::FILTERED);
+ }
+ }
+
+ if (output_file.length() > 0) {
+ pintool.write_coverages_as_pinlist(output_file, write_quota);
+ }
+
+ return 0;
+ } else {
+ if (probe_type != ProbeType::GENERATE) {
+ cerr << "Only generating probes is supported for non-zip files, please include "
+ "--gen-probe on your command"
+ << endl;
+ return 1;
+ }
+
+ // Generic file probing will just return resident memory and offsets
+ // without more contextual information.
+ VmaRangeGroup resident;
+
+ int res = pintool.probe_resident();
+ if (res > 0) {
+ cerr << "Failed to generate probe. Error Code: " << res << endl;
+ return 1;
+ }
+
+ pintool.dump_coverages(PinTool::DumpType::PROBE);
+
+ if (output_file.length() > 0) {
+ res = write_pinlist_file(output_file, resident.ranges, write_quota);
+ if (res > 0) {
+ cerr << "Failed to write pin file at: " << output_file << endl;
+ } else if (verbose) {
+ cout << "Finished writing pin file at: " << output_file << endl;
+ }
+ }
+ return res;
+ }
+ return 0;
+}
+
+int perform_pinlist_action(const vector<string>& options) {
+ string pinner_file;
+ bool verbose = false;
+ bool dump = false;
+ bool summary = false;
+
+ if (options.size() < 1) {
+ cerr << "Missing arguments for pinlist mode. See usage for details << endl";
+ return 1;
+ }
+ pinner_file = options[0];
+ for (int i = 1; i < options.size(); ++i) {
+ string option = options[i];
+
+ if (option == "-v") {
+ verbose = true;
+ }
+
+ if (option == "--dump") {
+ dump = true;
+ }
+
+ if (option == "--summary") {
+ summary = true;
+ }
+ }
+
+ if (pinner_file.empty()) {
+ cerr << "Error: Pinlist file to dump is missing. Specify it with '-p <file>'" << endl;
+ return 1;
+ }
+
+ if (verbose) {
+ cout << "Setting file to dump: " << pinner_file.c_str() << endl;
+ }
+
+ vector<VmaRange> vma_ranges;
+ if (read_pinlist_file(pinner_file, vma_ranges) == 1) {
+ cerr << "Failed reading pinlist file" << endl;
+ }
+
+ if (dump) {
+ print_pinlist_ranges(vma_ranges);
+ }
+
+ if (summary) {
+ print_pinlist_summary(vma_ranges);
+ }
+
+ return 0;
+}
+
+void print_usage() {
+ const string usage = R"(
+ Expected usage: pintool <mode> <required> [option]
+ where:
+ ./pintool <MODE>
+ <MODE>
+ file <filename> [option]
+ [option]
+ --gen-probe
+ Generate a probe from current resident memory based on provided "file"
+ --use-probe <path_to_input_pinlist.meta>
+ Use a previously generated pinlist.meta style file as the probe to match against.
+ --dump
+ Dump output contents to console.
+ --zip
+ Treat the file as a zip/apk file required for doing per-file coverage analysis and generation.
+ --pinconfig <path_to_pinconfig.txt>
+ Filter output coverage ranges using a provided pinconfig.txt style file. See README.md for samples
+ on the format of that file.
+ -v
+ Enable verbose output.
+
+ pinlist <pinlist_file> [option]
+ <pinlist_file>
+ this is the file that will be used for reading and it should follow the pinlist.meta format.
+ [option]
+ --dump
+ Dump <pinlist_file> contents to console output.
+ -v
+ Enable verbose output.
+ --summary
+ Summary results for the pinlist.meta file
+ )";
+ cout << usage.c_str();
+}
+
+int main(int argc, char** argv) {
+ if (argc == 1) {
+ print_usage();
+ return 0;
+ }
+
+ if (argc < 2) {
+ cerr << "<mode> is missing";
+ return 1;
+ }
+
+ if (strcmp(argv[1], "--help") == 0) {
+ print_usage();
+ return 0;
+ }
+
+ ToolMode mode = ToolMode::UNKNOWN;
+ if (strcmp(argv[1], "file") == 0) {
+ mode = ToolMode::MAPPED_FILE;
+ } else if (strcmp(argv[1], "pinlist") == 0) {
+ mode = ToolMode::PINLIST;
+ }
+
+ if (mode == ToolMode::UNKNOWN) {
+ cerr << "Failed to find mode: " << argv[1] << ". See usage for available modes." << endl;
+ return 1;
+ }
+
+ vector<string> options;
+ for (int i = 2; i < argc; ++i) {
+ options.push_back(argv[i]);
+ }
+
+ int res;
+ switch (mode) {
+ case ToolMode::MAPPED_FILE:
+ res = perform_file_action(options);
+ break;
+ case ToolMode::PINLIST:
+ res = perform_pinlist_action(options);
+ break;
+ case ToolMode::UNKNOWN:
+ cerr << "Unknown <MODE> see usage for details." << endl;
+ return 1;
+ }
+
+ return res;
+}
diff --git a/pinner/sample_pinconfig.txt b/pinner/sample_pinconfig.txt
new file mode 100644
index 00000000..3d9a13a6
--- /dev/null
+++ b/pinner/sample_pinconfig.txt
@@ -0,0 +1,5 @@
+file lib/arm64-v8a/libcrashpad_handler_trampoline.so
+file libmonochrome_64.so
+offset 108391596
+len 33348032
+file libarcore_sdk_c.so \ No newline at end of file
diff --git a/pinner/tests/Android.bp b/pinner/tests/Android.bp
new file mode 100644
index 00000000..f7b04d95
--- /dev/null
+++ b/pinner/tests/Android.bp
@@ -0,0 +1,95 @@
+// Copyright (C) 2018 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package {
+ default_applicable_licenses: ["Android-Apache-2.0"],
+}
+
+cc_test {
+ name: "meminspect_tests",
+
+ test_suites: ["device-tests"],
+
+ // Required for reading-writing files which are part of the tests.
+ require_root: true,
+
+ shared_libs: [
+ "libbase",
+ "libziparchive",
+ ],
+
+ cppflags: [
+ "-g",
+ "-Wall",
+ "-Werror",
+ ],
+
+ static_libs: [
+ "libmeminspect",
+ ],
+
+ target: {
+ android: {
+ srcs: ["meminspect_tests.cpp"],
+ },
+ },
+
+ cflags: [
+ "-Wall",
+ "-Wextra",
+ "-Werror",
+ "-O0", // as some tests rely on compiler keeping code as is
+ ],
+
+ compile_multilib: "first",
+}
+
+cc_test {
+ name: "pintool_tests",
+
+ test_suites: ["device-tests"],
+
+ // Required for reading-writing files which are part of the tests.
+ require_root: true,
+
+ shared_libs: [
+ "libbase",
+ "libziparchive",
+ ],
+
+ cppflags: [
+ "-g",
+ "-Wall",
+ "-Werror",
+ ],
+
+ static_libs: [
+ "libmeminspect",
+ ],
+
+ target: {
+ android: {
+ srcs: ["pintool_tests.cpp"],
+ },
+ },
+
+ cflags: [
+ "-Wall",
+ "-Wextra",
+ "-Werror",
+ "-O0", // as some tests rely on compiler keeping code as is
+ ],
+
+ compile_multilib: "first",
+}
diff --git a/pinner/tests/TEST_MAPPING b/pinner/tests/TEST_MAPPING
new file mode 100644
index 00000000..73af05f1
--- /dev/null
+++ b/pinner/tests/TEST_MAPPING
@@ -0,0 +1,10 @@
+{
+ "postsubmit": [
+ {
+ "name": "pintool_tests"
+ },
+ {
+ "name": "meminspect_tests"
+ }
+ ]
+} \ No newline at end of file
diff --git a/pinner/tests/meminspect_tests.cpp b/pinner/tests/meminspect_tests.cpp
new file mode 100644
index 00000000..98c51def
--- /dev/null
+++ b/pinner/tests/meminspect_tests.cpp
@@ -0,0 +1,284 @@
+#include <gtest/gtest.h>
+#include <stdio.h>
+
+#include <meminspect.h>
+
+using namespace std;
+
+/**
+ * This test is meant to be ran by directly pushing the test binary
+ * into the device as using atest will not provide sufficient privileges
+ * to execute drop_caches command.
+ */
+TEST(meminspect_test, inspect_matches_resident) {
+ // Create test file
+ string test_file = "/data/local/tmp/meminspect_test";
+ // If for any reason a test file already existed from a previous test, remove it.
+ remove(test_file.c_str());
+
+ int test_file_fd = open(test_file.c_str(), O_RDWR | O_CREAT, "w");
+ unsigned int page_size = sysconf(_SC_PAGESIZE);
+ if (test_file_fd == -1) {
+ ADD_FAILURE() << "Failed to open test file for writing. errno: " << std::strerror(errno);
+ close(test_file_fd);
+ remove(test_file.c_str());
+ return;
+ }
+
+ uint8_t* page_data = new uint8_t[page_size];
+ for (unsigned int i = 0; i < page_size; ++i) {
+ page_data[i] = i + 1;
+ }
+ int pages_to_write = 100;
+ for (int page = 0; page < pages_to_write; ++page) {
+ write(test_file_fd, page_data, page_size);
+ }
+ // fsync to ensure the data is flushed to disk.
+ if (fsync(test_file_fd) == -1) {
+ ADD_FAILURE() << "fsync failed errno: " << std::strerror(errno);
+ close(test_file_fd);
+ remove(test_file.c_str());
+ return;
+ }
+ close(test_file_fd);
+
+ // Drop the pagecache to ensure we do not have memory due to it staying there after write.
+ int drop_cache_fd = open("/proc/sys/vm/drop_caches", O_WRONLY);
+ if (drop_cache_fd == -1) {
+ ADD_FAILURE() << "failed opening drop caches fd errno: " << std::strerror(errno);
+ close(test_file_fd);
+ remove(test_file.c_str());
+ return;
+ }
+ write(drop_cache_fd, "3", 1);
+ fsync(drop_cache_fd);
+ close(drop_cache_fd);
+
+ // Open file and page in some memory
+ test_file_fd = open(test_file.c_str(), O_RDONLY, "r");
+ if (test_file_fd == -1) {
+ ADD_FAILURE() << "Failed to open test file for reading after creation. errno: "
+ << std::strerror(errno);
+ close(test_file_fd);
+ remove(test_file.c_str());
+ return;
+ }
+
+ char* base_address = (char*)mmap(0, page_size * pages_to_write, PROT_READ, MAP_SHARED,
+ test_file_fd, /*offset*/ 0);
+ if (base_address == (char*)-1) {
+ ADD_FAILURE() << "Failed to mmap file for reading after creation. errno: "
+ << std::strerror(errno);
+ close(test_file_fd);
+ remove(test_file.c_str());
+ return;
+ }
+
+ VmaRangeGroup vmas_resident;
+ int res = probe_resident_memory(test_file, vmas_resident, 1);
+ EXPECT_TRUE(res == 0);
+
+ // Probing the file without reading anything yields no resident memory
+ EXPECT_TRUE(vmas_resident.ranges.empty());
+
+ // Clear our to start fresh for next probe.
+ vmas_resident = VmaRangeGroup();
+
+ int pages_to_read = 1;
+ char* read_data = new char[pages_to_read];
+ for (int page = 0; page < pages_to_read; ++page) {
+ // Read 1 byte from each page to page it in.
+ read_data[page] = base_address[page * page_size];
+ }
+ res = probe_resident_memory(test_file, vmas_resident, 1);
+ EXPECT_TRUE(res == 0);
+
+ // The amount of memory paged in is outside our control, but we should have some.
+ uint64_t resident_total_size = vmas_resident.compute_total_size();
+ EXPECT_TRUE(resident_total_size > 0);
+ EXPECT_TRUE(vmas_resident.ranges.size() == 1);
+ EXPECT_TRUE(vmas_resident.ranges[0].offset == 0);
+ EXPECT_TRUE((uint64_t)vmas_resident.ranges[0].length == resident_total_size);
+
+ close(test_file_fd);
+ remove(test_file.c_str());
+}
+
+TEST(meminspect_test, custom_probe_coverage_matches_with_probe) {
+ ZipMemInspector inspector("");
+ VmaRangeGroup* probe = new VmaRangeGroup();
+ probe->ranges.push_back(VmaRange(0, 500));
+ probe->ranges.push_back(VmaRange(700, 100));
+ probe->ranges.push_back(VmaRange(1000, 500));
+ probe->ranges.push_back(VmaRange(2000, 100));
+ // Probed Resident Memory Offset ranges:
+ // [0,500],[700,800],[1000,1500],[2000,2100]
+ EXPECT_EQ(probe->compute_total_size(), (unsigned long long)1200);
+ inspector.set_existing_probe(probe);
+
+ // Emulate reading some files from the zip to compute their coverages
+ // fake1 memory offset ranges [100,300]
+ ZipEntryInfo info;
+ info.name = "fake1";
+ info.offset_in_zip = 100;
+ info.file_size_bytes = 200;
+ inspector.add_file_info(info);
+
+ // fake2 memory offset ranges [600,1200]
+ ZipEntryInfo info2;
+ info2.name = "fake2";
+ info2.offset_in_zip = 600;
+ info2.file_size_bytes = 600;
+ inspector.add_file_info(info2);
+
+ inspector.compute_per_file_coverage();
+ std::vector<ZipEntryCoverage> coverages = inspector.get_file_coverages();
+ EXPECT_EQ(coverages.size(), (size_t)2);
+
+ // Result coverage for fake1 should be: [100,300]
+ EXPECT_EQ(coverages[0].coverage.ranges[0].offset, (uint32_t)100);
+ EXPECT_EQ(coverages[0].coverage.ranges[0].length, (uint32_t)200);
+ EXPECT_EQ(coverages[0].coverage.compute_total_size(), (unsigned long long)200);
+ EXPECT_EQ(coverages[0].info.name, "fake1");
+ EXPECT_EQ(coverages[0].info.offset_in_zip, (uint32_t)100);
+ EXPECT_EQ(coverages[0].info.file_size_bytes, (uint32_t)200);
+
+ // coverage coverage for fake2 should be: [700,800] and [1000,1200]
+ EXPECT_EQ(coverages[1].coverage.ranges[0].offset, (uint32_t)700);
+ EXPECT_EQ(coverages[1].coverage.ranges[0].length, (uint32_t)100);
+ EXPECT_EQ(coverages[1].coverage.ranges[1].offset, (uint32_t)1000);
+ EXPECT_EQ(coverages[1].coverage.ranges[1].length, (uint32_t)200);
+ EXPECT_EQ(coverages[1].coverage.compute_total_size(), (unsigned long long)300); // 100 +
+ // 200
+ EXPECT_EQ(coverages[1].info.name, "fake2");
+ EXPECT_EQ(coverages[1].info.offset_in_zip, (uint32_t)600);
+ EXPECT_EQ(coverages[1].info.file_size_bytes, (uint32_t)600);
+}
+
+TEST(meminspect_test, whole_file_coverage_against_probe) {
+ ZipMemInspector inspector("");
+
+ // Emulate reading some files from the zip to compute their coverages
+ // fake1 memory offset ranges [100,300]
+ ZipEntryInfo info;
+ info.name = "fake1";
+ info.offset_in_zip = 100;
+ info.file_size_bytes = 200;
+ inspector.add_file_info(info);
+
+ // fake2 memory offset ranges [600,1200]
+ ZipEntryInfo info2;
+ info2.name = "fake2";
+ info2.offset_in_zip = 600;
+ info2.file_size_bytes = 600;
+ inspector.add_file_info(info2);
+
+ inspector.compute_per_file_coverage();
+ std::vector<ZipEntryCoverage> coverages = inspector.get_file_coverages();
+ EXPECT_EQ(coverages.size(), (size_t)2);
+
+ // Check that coverage matches entire file sizes
+ EXPECT_EQ(coverages[0].coverage.ranges[0].offset, (uint32_t)100);
+ EXPECT_EQ(coverages[0].coverage.ranges[0].length, (uint32_t)200);
+ EXPECT_EQ(coverages[0].coverage.compute_total_size(), (unsigned long long)200);
+ EXPECT_EQ(coverages[0].info.name, "fake1");
+ EXPECT_EQ(coverages[0].info.offset_in_zip, (uint32_t)100);
+ EXPECT_EQ(coverages[0].info.file_size_bytes, (uint32_t)200);
+
+ EXPECT_EQ(coverages[1].coverage.ranges[0].offset, (uint32_t)600);
+ EXPECT_EQ(coverages[1].coverage.ranges[0].length, (uint32_t)600);
+ EXPECT_EQ(coverages[1].coverage.compute_total_size(), (unsigned long long)600);
+ EXPECT_EQ(coverages[1].info.name, "fake2");
+ EXPECT_EQ(coverages[1].info.offset_in_zip, (uint32_t)600);
+ EXPECT_EQ(coverages[1].info.file_size_bytes, (uint32_t)600);
+}
+
+TEST(meminspect_test, file_multiple_ranges_matches_probe) {
+ VmaRangeGroup probe;
+ probe.ranges.push_back(VmaRange(0, 500));
+ probe.ranges.push_back(VmaRange(700, 100));
+ probe.ranges.push_back(VmaRange(1000, 500));
+ probe.ranges.push_back(VmaRange(2000, 100));
+ // Probed Resident Memory Offset ranges:
+ // [0,500],[700,800],[1000,1500],[2000,2100]
+ EXPECT_EQ(probe.compute_total_size(), (unsigned long long)1200);
+
+ std::vector<ZipEntryCoverage> desired_coverages;
+
+ // fake1 file resides between [100,1100]
+ // desired ranges are [100,200],[400,710],[820,850]
+ ZipEntryCoverage file1_mem;
+ file1_mem.info.name = "fake1";
+ file1_mem.info.offset_in_zip = 100;
+ file1_mem.info.file_size_bytes = 1000;
+ file1_mem.coverage.ranges.push_back(VmaRange(100, 100));
+ file1_mem.coverage.ranges.push_back(VmaRange(400, 310));
+ file1_mem.coverage.ranges.push_back(VmaRange(820, 30));
+ desired_coverages.push_back(file1_mem);
+
+ // fake2 memory offset ranges [1300,2100]
+ // desired ranges are [1400,1500],[1600,1650],[1800,2050]
+ ZipEntryCoverage file2_mem;
+ file2_mem.info.name = "fake2";
+ file2_mem.info.offset_in_zip = 1300;
+ file2_mem.info.file_size_bytes = 750;
+ file2_mem.coverage.ranges.push_back(VmaRange(1400, 100));
+ file2_mem.coverage.ranges.push_back(VmaRange(1600, 50));
+ file2_mem.coverage.ranges.push_back(VmaRange(1800, 250));
+ desired_coverages.push_back(file2_mem);
+
+ std::vector<ZipEntryCoverage> coverages =
+ ZipMemInspector::compute_coverage(desired_coverages, &probe);
+
+ EXPECT_EQ(coverages.size(), (size_t)2);
+
+ // Result coverage for fake1 should be: [100,200],[400,500],[700,710]
+ EXPECT_EQ(coverages[0].coverage.ranges[0].offset, (uint32_t)100);
+ EXPECT_EQ(coverages[0].coverage.ranges[0].length, (uint32_t)100);
+ EXPECT_EQ(coverages[0].coverage.ranges[1].offset, (uint32_t)400);
+ EXPECT_EQ(coverages[0].coverage.ranges[1].length, (uint32_t)100);
+ EXPECT_EQ(coverages[0].coverage.ranges[2].offset, (uint32_t)700);
+ EXPECT_EQ(coverages[0].coverage.ranges[2].length, (uint32_t)10);
+
+ EXPECT_EQ(coverages[0].coverage.compute_total_size(), (unsigned long long)210);
+ EXPECT_EQ(coverages[0].info.name, "fake1");
+ EXPECT_EQ(coverages[0].info.offset_in_zip, (uint32_t)100);
+ EXPECT_EQ(coverages[0].info.file_size_bytes, (uint32_t)1000);
+
+ // coverage coverage for fake2 should be: [1400,1500],[2000,2050]
+ EXPECT_EQ(coverages[1].coverage.ranges[0].offset, (uint32_t)1400);
+ EXPECT_EQ(coverages[1].coverage.ranges[0].length, (uint32_t)100);
+ EXPECT_EQ(coverages[1].coverage.ranges[1].offset, (uint32_t)2000);
+ EXPECT_EQ(coverages[1].coverage.ranges[1].length, (uint32_t)50);
+ EXPECT_EQ(coverages[1].coverage.compute_total_size(), (unsigned long long)150);
+ EXPECT_EQ(coverages[1].info.name, "fake2");
+ EXPECT_EQ(coverages[1].info.offset_in_zip, (uint32_t)1300);
+ EXPECT_EQ(coverages[1].info.file_size_bytes, (uint32_t)750);
+}
+
+TEST(meminspect_test, range_alignment_and_merge_matches) {
+ ZipMemInspector inspector("");
+ VmaRangeGroup* probe = new VmaRangeGroup();
+ probe->ranges.push_back(VmaRange(0, 500));
+ probe->ranges.push_back(VmaRange(700, 100));
+ int page_size = 4096;
+
+ // Probed Resident Memory Offset ranges:
+ // [0,500],[700,800]
+ inspector.set_existing_probe(probe);
+
+ // When we page align, we should end up with [0,500],[0,800]
+ align_ranges(probe->ranges, page_size);
+ EXPECT_EQ(probe->ranges[0].offset, (uint32_t)0);
+ EXPECT_EQ(probe->ranges[0].length, (uint32_t)500);
+ EXPECT_EQ(probe->ranges[1].offset, (uint32_t)0);
+ EXPECT_EQ(probe->ranges[1].length, (uint32_t)800);
+ EXPECT_EQ(probe->ranges.size(), (uint32_t)2);
+
+ // Because we have overlapping ranges, a union-merge should
+ // skip duplication of intersections and end up with [0,800]
+ std::vector<VmaRange> merged = merge_ranges(probe->ranges);
+ EXPECT_EQ(merged[0].offset, (uint32_t)0);
+ EXPECT_EQ(merged[0].length, (uint32_t)800);
+ EXPECT_EQ(merged.size(), (uint32_t)1);
+} \ No newline at end of file
diff --git a/pinner/tests/pintool_tests.cpp b/pinner/tests/pintool_tests.cpp
new file mode 100644
index 00000000..f351a90a
--- /dev/null
+++ b/pinner/tests/pintool_tests.cpp
@@ -0,0 +1,137 @@
+#include <gtest/gtest.h>
+#include <stdio.h>
+#include <list>
+
+#include <pin_utils.h>
+
+using namespace std;
+
+TEST(pintool_test, pinlist_matches_memranges) {
+ vector<VmaRange> vma_ranges;
+ unsigned int page_size = sysconf(_SC_PAGESIZE);
+ vma_ranges.push_back(VmaRange(0, 500));
+ vma_ranges.push_back(VmaRange(5000, 5500));
+ vma_ranges.push_back(VmaRange(21000, 13000));
+ vma_ranges.push_back(VmaRange(50000, 35000));
+
+ string test_file = "/data/local/tmp/pintool_test";
+ write_pinlist_file(test_file, vma_ranges);
+
+ vector<VmaRange> read_ranges;
+ read_pinlist_file(test_file, read_ranges);
+
+ EXPECT_EQ(vma_ranges.size(), read_ranges.size());
+ for (size_t i = 0; i < vma_ranges.size(); ++i) {
+ // We expect to write pinlists that are page-aligned, so
+ // we compare against page aligned offsets.
+ uint64_t unaligned_bytes = vma_ranges[i].offset % page_size;
+ EXPECT_EQ(vma_ranges[i].offset - unaligned_bytes, read_ranges[i].offset);
+ EXPECT_EQ(vma_ranges[i].length + unaligned_bytes, read_ranges[i].length);
+ }
+
+ remove(test_file.c_str());
+}
+
+TEST(pintool_test, pinlist_quota_applied) {
+ vector<VmaRange> vma_ranges;
+ unsigned int page_size = sysconf(_SC_PAGESIZE);
+ vma_ranges.push_back(VmaRange(0, 100));
+ vma_ranges.push_back(VmaRange(page_size, 500));
+ vma_ranges.push_back(VmaRange(page_size * 2, 300));
+ vma_ranges.push_back(VmaRange(page_size * 3, 200));
+
+ const int ranges_to_write = 700;
+ string test_file = "/data/local/tmp/pintool_test";
+ write_pinlist_file(test_file, vma_ranges, ranges_to_write);
+
+ vector<VmaRange> read_ranges;
+ read_pinlist_file(test_file, read_ranges);
+
+ int total_length = 0;
+ for (size_t i = 0; i < read_ranges.size(); ++i) {
+ total_length += read_ranges[i].length;
+ }
+ EXPECT_EQ(total_length, ranges_to_write);
+
+ remove(test_file.c_str());
+}
+
+TEST(pintool_test, pinconfig_filter_coverage_matches) {
+ VmaRangeGroup* probe = new VmaRangeGroup();
+ probe->ranges.push_back(VmaRange(0, 500));
+ probe->ranges.push_back(VmaRange(1000, 5000));
+
+ ZipMemInspector* inspector = new ZipMemInspector("");
+
+ // Probed Resident Memory Offset ranges:
+ // [0,500],[1000,6000]
+ probe->compute_total_size();
+ inspector->set_existing_probe(probe);
+
+ // Emulate reading some files from the zip to compute their coverages
+ // fake1 memory offset ranges [100,400]
+ ZipEntryInfo info;
+ info.name = "fake1";
+ info.offset_in_zip = 100;
+ info.file_size_bytes = 300;
+ inspector->add_file_info(info);
+
+ // fake2 memory offset ranges [600,3000]
+ ZipEntryInfo info2;
+ info2.name = "fake2";
+ info2.offset_in_zip = 600;
+ info2.file_size_bytes = 2400;
+ inspector->add_file_info(info2);
+
+ ZipEntryInfo info3;
+ info2.name = "fake3";
+ info2.offset_in_zip = 3100;
+ info2.file_size_bytes = 200;
+ inspector->add_file_info(info3);
+
+ // Create a fake pinconfig file
+ PinConfig* pinconfig = new PinConfig();
+
+ // First file we want it entirely so don't provide ranges
+ PinConfigFile pinconfig_file_1;
+ pinconfig_file_1.filename = "fake1";
+ pinconfig->files_.push_back(pinconfig_file_1);
+
+ // Add a partially matched file
+ PinConfigFile pinconfig_file_2;
+ pinconfig_file_2.filename = "fake2";
+ pinconfig_file_2.ranges.push_back(VmaRange(100, 500));
+ pinconfig_file_2.ranges.push_back(VmaRange(800, 200));
+ pinconfig->files_.push_back(pinconfig_file_2);
+
+ // Add a file that does not exist
+ PinConfigFile pinconfig_file_3;
+ pinconfig_file_3.filename = "fake4";
+ pinconfig_file_3.ranges.push_back(VmaRange(0, 1000));
+ pinconfig->files_.push_back(pinconfig_file_3);
+
+ PinTool pintool("");
+ pintool.set_custom_zip_inspector(inspector);
+ pintool.compute_zip_entry_coverages();
+ pintool.filter_zip_entry_coverages(pinconfig);
+
+ std::vector<ZipEntryCoverage> filtered = pintool.get_filtered_zip_entries();
+
+ // We only matched 2 files, one should not have matched to any filter.
+ EXPECT_EQ(filtered.size(), (unsigned long)2);
+ EXPECT_EQ(filtered[0].info.name, "fake1");
+ EXPECT_EQ(filtered[0].coverage.ranges[0].offset, (unsigned long)100);
+ EXPECT_EQ(filtered[0].coverage.ranges[0].length, (unsigned long)300);
+
+ // Probe Resident has [0,500],[1000,6000].
+ // fake2 file lives within [600,3000]
+ // fake2 relative offsets from pinconfig [100,600],[800,1000]
+ // fake2 absolute zip offsets are [700,1200],[1400,1600]
+ // then matching absolute offsets against resident yields [1000,1200],[1400,1600]
+ EXPECT_EQ(filtered[1].info.name, "fake2");
+ EXPECT_EQ(filtered[1].info.offset_in_zip, (unsigned long)600);
+ EXPECT_EQ(filtered[1].coverage.ranges[0].offset, (unsigned long)1000);
+ EXPECT_EQ(filtered[1].coverage.ranges[0].length, (unsigned long)200);
+ EXPECT_EQ(filtered[1].coverage.ranges[1].offset, (unsigned long)1400);
+ EXPECT_EQ(filtered[1].coverage.ranges[1].length, (unsigned long)200);
+} \ No newline at end of file
diff --git a/profcollectd/libprofcollectd/config.rs b/profcollectd/libprofcollectd/config.rs
index 14236ab9..87242489 100644
--- a/profcollectd/libprofcollectd/config.rs
+++ b/profcollectd/libprofcollectd/config.rs
@@ -30,7 +30,8 @@ use std::time::Duration;
const PROFCOLLECT_CONFIG_NAMESPACE: &str = "profcollect_native_boot";
const PROFCOLLECT_NODE_ID_PROPERTY: &str = "persist.profcollectd.node_id";
-const DEFAULT_BINARY_FILTER: &str = "^/(system|apex/.+)/(bin|lib|lib64)/.+";
+const DEFAULT_BINARY_FILTER: &str =
+ "(^/(system|apex/.+|vendor)/(bin|lib|lib64)/.+)|kernel.kallsyms";
pub const REPORT_RETENTION_SECS: u64 = 14 * 24 * 60 * 60; // 14 days.
// Static configs that cannot be changed.