diff options
author | Collin Fijalkovich <cfijalkovich@google.com> | 2021-05-10 19:57:27 +0000 |
---|---|---|
committer | Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com> | 2021-05-10 19:57:27 +0000 |
commit | be2927bf2f6b6378af112e94602f3387a8524e6a (patch) | |
tree | 35333fe8d7ab03b660b2d5e3bad5b7d2c3a3c7e5 | |
parent | 9f53cf2c62225fcd1ab4c50b8481053ab25b0a97 (diff) | |
parent | 20ab1b7b47a417654c329a7781bd12e30879ee32 (diff) | |
download | bionic-be2927bf2f6b6378af112e94602f3387a8524e6a.tar.gz |
Merge "Bionic: Mark PMD aligned text segments huge page eligible" am: 20ab1b7b47
Original change: https://android-review.googlesource.com/c/platform/bionic/+/1652592
Change-Id: I9ea5cd0ba509d44081f92fbf59163113a87f5f38
-rw-r--r-- | linker/Android.bp | 1 | ||||
-rw-r--r-- | linker/NOTICE | 28 | ||||
-rw-r--r-- | linker/linker.h | 2 | ||||
-rw-r--r-- | linker/linker_main.cpp | 3 | ||||
-rw-r--r-- | linker/linker_phdr.cpp | 75 | ||||
-rw-r--r-- | linker/linker_phdr.h | 2 | ||||
-rw-r--r-- | linker/linker_transparent_hugepage_support.cpp | 44 |
7 files changed, 140 insertions, 15 deletions
diff --git a/linker/Android.bp b/linker/Android.bp index 3370f2826..284da5821 100644 --- a/linker/Android.bp +++ b/linker/Android.bp @@ -193,6 +193,7 @@ filegroup { "linker_relocate.cpp", "linker_sdk_versions.cpp", "linker_soinfo.cpp", + "linker_transparent_hugepage_support.cpp", "linker_tls.cpp", "linker_utils.cpp", "rt.cpp", diff --git a/linker/NOTICE b/linker/NOTICE index 4cada69c0..8f70d871d 100644 --- a/linker/NOTICE +++ b/linker/NOTICE @@ -306,3 +306,31 @@ SUCH DAMAGE. ------------------------------------------------------------------- +Copyright (C) 2021 The Android Open Source Project +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS +OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +SUCH DAMAGE. + +------------------------------------------------------------------- + diff --git a/linker/linker.h b/linker/linker.h index e1775fb66..74bdcc7fa 100644 --- a/linker/linker.h +++ b/linker/linker.h @@ -110,6 +110,8 @@ void ___cfi_fail(uint64_t CallSiteTypeId, void* Ptr, void *DiagData, void *Ret); void set_application_target_sdk_version(int target); int get_application_target_sdk_version(); +bool get_transparent_hugepages_supported(); + enum { /* A regular namespace is the namespace with a custom search path that does * not impose any restrictions on the location of native libraries. diff --git a/linker/linker_main.cpp b/linker/linker_main.cpp index 0b501a7b1..2a690e9a0 100644 --- a/linker/linker_main.cpp +++ b/linker/linker_main.cpp @@ -31,9 +31,10 @@ #include <link.h> #include <sys/auxv.h> +#include "linker.h" +#include "linker_cfi.h" #include "linker_debug.h" #include "linker_debuggerd.h" -#include "linker_cfi.h" #include "linker_gdb_support.h" #include "linker_globals.h" #include "linker_phdr.h" diff --git a/linker/linker_phdr.cpp b/linker/linker_phdr.cpp index 9b1b99ffd..60fd77620 100644 --- a/linker/linker_phdr.cpp +++ b/linker/linker_phdr.cpp @@ -70,8 +70,9 @@ static int GetTargetElfMachine() { p_memsz -> segment memory size (always >= p_filesz) p_vaddr -> segment's virtual address p_flags -> segment flags (e.g. readable, writable, executable) + p_align -> segment's in-memory and in-file alignment - We will ignore the p_paddr and p_align fields of ElfW(Phdr) for now. + We will ignore the p_paddr field of ElfW(Phdr) for now. The loadable segments can be seen as a list of [p_vaddr ... p_vaddr+p_memsz) ranges of virtual addresses. A few rules apply: @@ -137,6 +138,9 @@ static int GetTargetElfMachine() { MAYBE_MAP_FLAG((x), PF_R, PROT_READ) | \ MAYBE_MAP_FLAG((x), PF_W, PROT_WRITE)) +// Default PMD size for x86_64 and aarch64 (2MB). +static constexpr size_t kPmdSize = (1UL << 21); + ElfReader::ElfReader() : did_read_(false), did_load_(false), fd_(-1), file_offset_(0), file_size_(0), phdr_num_(0), phdr_table_(nullptr), shdr_table_(nullptr), shdr_num_(0), dynamic_(nullptr), strtab_(nullptr), @@ -526,12 +530,40 @@ size_t phdr_table_get_load_size(const ElfW(Phdr)* phdr_table, size_t phdr_count, return max_vaddr - min_vaddr; } +// Returns the maximum p_align associated with a loadable segment in the ELF +// program header table. Used to determine whether the file should be loaded at +// a specific virtual address alignment for use with huge pages. +size_t phdr_table_get_maximum_alignment(const ElfW(Phdr)* phdr_table, size_t phdr_count) { + size_t maximum_alignment = PAGE_SIZE; + + for (size_t i = 0; i < phdr_count; ++i) { + const ElfW(Phdr)* phdr = &phdr_table[i]; + + // p_align must be 0, 1, or a positive, integral power of two. + if (phdr->p_type != PT_LOAD || ((phdr->p_align & (phdr->p_align - 1)) != 0)) { + continue; + } + + if (phdr->p_align > maximum_alignment) { + maximum_alignment = phdr->p_align; + } + } + +#if defined(__LP64__) + return maximum_alignment; +#else + return PAGE_SIZE; +#endif +} + // Reserve a virtual address range such that if it's limits were extended to the next 2**align // boundary, it would not overlap with any existing mappings. -static void* ReserveWithAlignmentPadding(size_t size, size_t align, void** out_gap_start, - size_t* out_gap_size) { +static void* ReserveWithAlignmentPadding(size_t size, size_t mapping_align, size_t start_align, + void** out_gap_start, size_t* out_gap_size) { int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS; - if (align == PAGE_SIZE) { + // Reserve enough space to properly align the library's start address. + mapping_align = std::max(mapping_align, start_align); + if (mapping_align == PAGE_SIZE) { void* mmap_ptr = mmap(nullptr, size, PROT_NONE, mmap_flags, -1, 0); if (mmap_ptr == MAP_FAILED) { return nullptr; @@ -550,24 +582,24 @@ static void* ReserveWithAlignmentPadding(size_t size, size_t align, void** out_g constexpr size_t kMaxGapUnits = 32; // Allocate enough space so that the end of the desired region aligned up is still inside the // mapping. - size_t mmap_size = align_up(size, align) + align - PAGE_SIZE; + size_t mmap_size = align_up(size, mapping_align) + mapping_align - PAGE_SIZE; uint8_t* mmap_ptr = reinterpret_cast<uint8_t*>(mmap(nullptr, mmap_size, PROT_NONE, mmap_flags, -1, 0)); if (mmap_ptr == MAP_FAILED) { return nullptr; } size_t gap_size = 0; - size_t first_byte = reinterpret_cast<size_t>(align_up(mmap_ptr, align)); - size_t last_byte = reinterpret_cast<size_t>(align_down(mmap_ptr + mmap_size, align) - 1); + size_t first_byte = reinterpret_cast<size_t>(align_up(mmap_ptr, mapping_align)); + size_t last_byte = reinterpret_cast<size_t>(align_down(mmap_ptr + mmap_size, mapping_align) - 1); if (kGapAlignment && first_byte / kGapAlignment != last_byte / kGapAlignment) { // This library crosses a 2MB boundary and will fragment a new huge page. // Lets take advantage of that and insert a random number of inaccessible huge pages before that // to improve address randomization and make it harder to locate this library code by probing. munmap(mmap_ptr, mmap_size); - align = std::max(align, kGapAlignment); + mapping_align = std::max(mapping_align, kGapAlignment); gap_size = kGapAlignment * (is_first_stage_init() ? 1 : arc4random_uniform(kMaxGapUnits - 1) + 1); - mmap_size = align_up(size + gap_size, align) + align - PAGE_SIZE; + mmap_size = align_up(size + gap_size, mapping_align) + mapping_align - PAGE_SIZE; mmap_ptr = reinterpret_cast<uint8_t*>(mmap(nullptr, mmap_size, PROT_NONE, mmap_flags, -1, 0)); if (mmap_ptr == MAP_FAILED) { return nullptr; @@ -582,13 +614,13 @@ static void* ReserveWithAlignmentPadding(size_t size, size_t align, void** out_g gap_start = gap_end = mmap_ptr + mmap_size; } - uint8_t* first = align_up(mmap_ptr, align); - uint8_t* last = align_down(gap_start, align) - size; + uint8_t* first = align_up(mmap_ptr, mapping_align); + uint8_t* last = align_down(gap_start, mapping_align) - size; // arc4random* is not available in first stage init because /dev/urandom hasn't yet been // created. Don't randomize then. - size_t n = is_first_stage_init() ? 0 : arc4random_uniform((last - first) / PAGE_SIZE + 1); - uint8_t* start = first + n * PAGE_SIZE; + size_t n = is_first_stage_init() ? 0 : arc4random_uniform((last - first) / start_align + 1); + uint8_t* start = first + n * start_align; // Unmap the extra space around the allocation. // Keep it mapped PROT_NONE on 64-bit targets where address space is plentiful to make it harder // to defeat ASLR by probing for readable memory mappings. @@ -622,7 +654,15 @@ bool ElfReader::ReserveAddressSpace(address_space_params* address_space) { load_size_ - address_space->reserved_size, load_size_, name_.c_str()); return false; } - start = ReserveWithAlignmentPadding(load_size_, kLibraryAlignment, &gap_start_, &gap_size_); + size_t start_alignment = PAGE_SIZE; + if (get_transparent_hugepages_supported() && get_application_target_sdk_version() >= 31) { + size_t maximum_alignment = phdr_table_get_maximum_alignment(phdr_table_, phdr_num_); + // Limit alignment to PMD size as other alignments reduce the number of + // bits available for ASLR for no benefit. + start_alignment = maximum_alignment == kPmdSize ? kPmdSize : PAGE_SIZE; + } + start = ReserveWithAlignmentPadding(load_size_, kLibraryAlignment, start_alignment, &gap_start_, + &gap_size_); if (start == nullptr) { DL_ERR("couldn't reserve %zd bytes of address space for \"%s\"", load_size_, name_.c_str()); return false; @@ -706,6 +746,13 @@ bool ElfReader::LoadSegments() { DL_ERR("couldn't map \"%s\" segment %zd: %s", name_.c_str(), i, strerror(errno)); return false; } + + // Mark segments as huge page eligible if they meet the requirements + // (executable and PMD aligned). + if ((phdr->p_flags & PF_X) && phdr->p_align == kPmdSize && + get_transparent_hugepages_supported()) { + madvise(seg_addr, file_length, MADV_HUGEPAGE); + } } // if the segment is writable, and does not end on a page boundary, diff --git a/linker/linker_phdr.h b/linker/linker_phdr.h index 548dc51dd..98bf020ac 100644 --- a/linker/linker_phdr.h +++ b/linker/linker_phdr.h @@ -120,6 +120,8 @@ class ElfReader { size_t phdr_table_get_load_size(const ElfW(Phdr)* phdr_table, size_t phdr_count, ElfW(Addr)* min_vaddr = nullptr, ElfW(Addr)* max_vaddr = nullptr); +size_t phdr_table_get_maximum_alignment(const ElfW(Phdr)* phdr_table, size_t phdr_count); + int phdr_table_protect_segments(const ElfW(Phdr)* phdr_table, size_t phdr_count, ElfW(Addr) load_bias, const GnuPropertySection* prop = nullptr); diff --git a/linker/linker_transparent_hugepage_support.cpp b/linker/linker_transparent_hugepage_support.cpp new file mode 100644 index 000000000..65ba4cd89 --- /dev/null +++ b/linker/linker_transparent_hugepage_support.cpp @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include <string> + +#include <android-base/file.h> + +#include "linker.h" + +bool get_transparent_hugepages_supported() { + static bool transparent_hugepages_supported = []() { + std::string enabled; + if (!android::base::ReadFileToString("/sys/kernel/mm/transparent_hugepage/enabled", &enabled)) { + return false; + } + return enabled.find("[never]") == std::string::npos; + }; + return transparent_hugepages_supported; +} |