#include "pin_utils.h" #include #include #include #include #include #include #include using namespace std; using namespace android::base; int write_pinlist_file(const std::string& output_file, const std::vector& files_to_write, int64_t write_quota) { std::vector ranges; for (auto&& file : files_to_write) { ranges.insert(ranges.end(), file.coverage.ranges.begin(), file.coverage.ranges.end()); } return write_pinlist_file(output_file, ranges, write_quota); } int write_pinlist_file(const std::string& output_file, const std::vector& vmas_to_write, int64_t write_quota) { ofstream pinlist_file(output_file); if (pinlist_file.fail()) { return 1; } int64_t total_written = 0; unsigned int page_size = sysconf(_SC_PAGESIZE); const bool has_quota = write_quota > 0; bool reached_quota = false; // The PinnerService does not require aligned offsets, however, aligning // allows our summary results to be accurate and avoids over-accounting // of pinning in PinnerService. std::vector processed_vmas_to_write = vmas_to_write; align_ranges(processed_vmas_to_write, page_size); // When we page-align the ranges, we may cause overlaps between ranges // as we elongate the begin offset to match the page the previous // range may end up overlapping the current one. processed_vmas_to_write = merge_ranges(processed_vmas_to_write); for (auto&& processed_vma_to_write : processed_vmas_to_write) { uint32_t vma_start_offset = processed_vma_to_write.offset; uint32_t vma_length = processed_vma_to_write.length; if (has_quota && (total_written + vma_length > write_quota)) { // We would go beyond quota, set the maximum allowed write and exit. vma_length = write_quota - total_written; reached_quota = true; } // Transform to BigEndian as PinnerService requires that endianness for reading. uint32_t vma_start_offset_be = htobe32(vma_start_offset); uint32_t vma_length_be = htobe32(vma_length); cout << "Pinlist Writing start=" << vma_start_offset << " bytes=" << vma_length << endl; pinlist_file.write(reinterpret_cast(&vma_start_offset_be), sizeof(vma_start_offset_be)); if (pinlist_file.fail()) { return 1; } pinlist_file.write(reinterpret_cast(&vma_length_be), sizeof(vma_length_be)); total_written += vma_length; if (pinlist_file.fail()) { return 1; } if (reached_quota) { break; } } return 0; } int read_pinlist_file(const std::string& pinner_file, /*out*/ std::vector& pinranges) { ifstream pinlist_file(pinner_file); if (pinlist_file.fail()) { return 1; } uint32_t vma_start; uint32_t vma_length; while (!pinlist_file.eof()) { pinlist_file.read(reinterpret_cast(&vma_start), sizeof(vma_start)); pinlist_file.read(reinterpret_cast(&vma_length), sizeof(vma_length)); if (pinlist_file.fail()) { return 1; } vma_start = betoh32(vma_start); vma_length = betoh32(vma_length); pinranges.push_back(VmaRange(vma_start, vma_length)); } return 0; } ZipEntryCoverage PinConfigFile::to_zipfilemem(const ZipEntryInfo& info) { ZipEntryCoverage file; file.info = info; if (ranges.empty()) { cout << "No ranges found for file " << info.name << " creating entire file range" << endl; // Any file coming from pinconfig without explicit // ranges will be assumed to be wanted in its entirety ranges.push_back(VmaRange(0, info.file_size_bytes)); } file.coverage.ranges = ranges; // Offsets specified in pinconfig file are relative to the file // so transform to zip global offsets which are used for coverage // computations. file.coverage.apply_offset(info.offset_in_zip); file.coverage.compute_total_size(); return file; } int PinConfig::parse(std::string config_file, bool verbose) { ifstream file(config_file); string file_in_zip; if (verbose) { cout << "Parsing file: " << config_file << endl; } string token; file >> token; while (!file.eof()) { if (token == "file") { file >> file_in_zip; PinConfigFile pin_config_file; pin_config_file.filename = file_in_zip; file >> token; while (token != "file" && !file.eof()) { VmaRange range; // Inner parsing loop for per file config. if (token == "offset") { file >> token; android::base::ParseUint(token, &range.offset); file >> token; if (token != "len") { cerr << "Malformed file, expected 'len' after offset" << endl; return 1; } file >> token; android::base::ParseUint(token, &range.length); pin_config_file.ranges.push_back(range); } file >> token; } files_.push_back(pin_config_file); } else { cerr << "Unexpected token: " << token << ". Exit read" << endl; return 1; } } if (files_.empty()) { cerr << "Failed parsing pinconfig file, no entries found." << endl; return 1; } if (verbose) { cout << "Finished parsing Pinconfig file" << endl; for (auto&& pin_file : files_) { cout << "file=" << pin_file.filename << endl; for (auto&& range : pin_file.ranges) { cout << "offset=" << range.offset << " bytes=" << range.length << endl; } } } return 0; } void PinTool::set_custom_zip_inspector(ZipMemInspector* inspector) { delete zip_inspector_; zip_inspector_ = inspector; } void PinTool::set_verbose_output(bool verbose) { verbose_ = verbose; } void PinTool::read_probe_from_pinlist(std::string custom_probe_file) { custom_probe_file_ = custom_probe_file; VmaRangeGroup* custom_probe = new VmaRangeGroup(); read_pinlist_file(custom_probe_file_, custom_probe->ranges); custom_probe->compute_total_size(); if (custom_probe->ranges.empty()) { cerr << "Did not find any memory range in " << custom_probe_file_ << endl; delete custom_probe; return; } zip_inspector_->set_existing_probe(custom_probe); } int PinTool::probe_resident() { return zip_inspector_->probe_resident(); } void PinTool::compute_zip_entry_coverages() { zip_inspector_->compute_per_file_coverage(); if (verbose_) { std::vector files = zip_inspector_->get_file_infos(); for (auto&& file : files) { cout << "file found. name=" << file.name << " offset=" << file.offset_in_zip << " uncompressed=" << file.uncompressed_size << " compressed=" << file.file_size_bytes << endl << endl; } } } void PinTool::dump_coverages(PinTool::DumpType dump_type) { std::vector* file_coverages; if (dump_type == PinTool::DumpType::FILTERED) { file_coverages = &filtered_files_; } else if (dump_type == PinTool::DumpType::FILE_COVERAGE) { file_coverages = &(zip_inspector_->get_file_coverages()); } else { // PinTool::DumpType::PROBE VmaRangeGroup* probe = zip_inspector_->get_probe(); file_coverages = new vector(); ZipEntryCoverage file; file.coverage = *probe; file.info.name = input_file_; file.info.offset_in_zip = 0; uint64_t file_size_bytes = get_file_size(input_file_); if (file_size_bytes == -1) { cerr << "Failed to dump, cannot fstat file: " << input_file_ << endl; delete file_coverages; return; } file.info.file_size_bytes = file_size_bytes; file_coverages->push_back(file); } for (auto&& file : *file_coverages) { uint64_t total_size = file.coverage.compute_total_size(); cout << file.info.name << " size(B)=" << file.info.file_size_bytes << " resident(B)=" << total_size << " resident(%)=" << (double)(total_size) / file.info.file_size_bytes * 100.0 << endl; if (verbose_) { cout << "file_base_zip_offset=" << file.info.offset_in_zip << endl; } cout << "file resident ranges" << endl; if (dump_type != DumpType::PROBE) { for (auto&& range : file.coverage.ranges) { // The offset in the range represents the absolute absolute offset relative to the // zip so substract the file base offset to get the relative offset within the file // which may be what is worth for a user to specify in pinconfig.txt files. uint64_t offset_in_file = range.offset - file.info.offset_in_zip; cout << "zip_offset=" << range.offset << " file_offset=" << offset_in_file << " total_bytes=" << range.length << endl; } } else { for (auto&& range : file.coverage.ranges) { cout << "file_offset=" << range.offset << " total_bytes=" << range.length << endl; } } cout << endl; } cout << endl; if (dump_type == DumpType::PROBE) { // For other dump types we do not create memory, we reuse from class. delete file_coverages; } } void PinTool::filter_zip_entry_coverages(const std::string& pinconfig_filename) { if (pinconfig_filename.length() == 0) { // Nothing to do. return; } PinConfig* pinconfig = new PinConfig(); if (pinconfig->parse(pinconfig_filename, verbose_) > 0) { cerr << "Failed parsing pinconfig file " << pinconfig_filename << ". Skip filtering"; delete pinconfig; return; } filter_zip_entry_coverages(pinconfig); } void PinTool::filter_zip_entry_coverages(PinConfig* pinconfig) { pinconfig_ = pinconfig; // Filter based on the per file configuration. vector file_coverages = zip_inspector_->get_file_coverages(); vector& filtered_files = filtered_files_; for (auto&& file_coverage : file_coverages) { for (auto&& pinconfig_file : pinconfig_->files_) { // Match each zip entry against every pattern in filter file. std::string_view file_coverage_view(file_coverage.info.name.c_str()); std::string_view pinconfig_view(pinconfig_file.filename.c_str()); if (file_coverage_view.find(pinconfig_view) != std::string_view::npos) { // Now that we found a match, create a file with offsets that are global to zip file ZipEntryCoverage file_in_config = pinconfig_file.to_zipfilemem(file_coverage.info); if (verbose_) { cout << "Found a match: file=" << file_coverage.info.name << " matching filter=" << pinconfig_file.filename << endl; for (auto&& range : file_in_config.coverage.ranges) { cout << "zip_offset=" << range.offset << " bytes=" << range.length << endl; } } ZipEntryCoverage filtered_file = file_coverage.compute_coverage(file_in_config.coverage); filtered_files.push_back(filtered_file); break; } } } } std::vector PinTool::get_filtered_zip_entries() { return filtered_files_; } void PinTool::write_coverages_as_pinlist(std::string output_pinlist, int64_t write_quota) { std::vector* pinlist_coverages = nullptr; if (!filtered_files_.empty()) { // Highest preference is writing filtered files if they exist if (verbose_) { cout << "Writing pinconfig filtered file coverages" << endl; } pinlist_coverages = &filtered_files_; } else if (!zip_inspector_->get_file_coverages().empty()) { // Fallback to looking for file coverage computation pinlist_coverages = &zip_inspector_->get_file_coverages(); if (verbose_) { cout << "Writing regular file coverages." << endl; } } if (pinlist_coverages == nullptr) { cerr << "Failed to find coverage to write to: " << output_pinlist << endl; return; } int res = write_pinlist_file(output_pinlist, *pinlist_coverages, write_quota); if (res > 0) { cerr << "Failed to write pin file at: " << output_pinlist << endl; } else { if (verbose_) { cout << "Finished writing pin file at: " << output_pinlist << endl; } } }