1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
|
/*
* Copyright (C) 2014 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "private/bionic_globals.h"
#include "private/bionic_vdso.h"
#include <limits.h>
#include <link.h>
#include <string.h>
#include <sys/auxv.h>
#include <sys/cdefs.h>
#include <sys/hwprobe.h>
#include <sys/time.h>
#include <syscall.h>
#include <time.h>
#include <unistd.h>
extern "C" int __clock_gettime(int, struct timespec*);
extern "C" int __clock_getres(int, struct timespec*);
extern "C" int __gettimeofday(struct timeval*, struct timezone*);
static inline int vdso_return(int result) {
if (__predict_true(result == 0)) return 0;
errno = -result;
return -1;
}
int clock_gettime(int clock_id, timespec* tp) {
auto vdso_clock_gettime = reinterpret_cast<decltype(&clock_gettime)>(
__libc_globals->vdso[VDSO_CLOCK_GETTIME].fn);
if (__predict_true(vdso_clock_gettime)) {
return vdso_return(vdso_clock_gettime(clock_id, tp));
}
return __clock_gettime(clock_id, tp);
}
int clock_getres(int clock_id, timespec* tp) {
auto vdso_clock_getres = reinterpret_cast<decltype(&clock_getres)>(
__libc_globals->vdso[VDSO_CLOCK_GETRES].fn);
if (__predict_true(vdso_clock_getres)) {
return vdso_return(vdso_clock_getres(clock_id, tp));
}
return __clock_getres(clock_id, tp);
}
int gettimeofday(timeval* tv, struct timezone* tz) {
auto vdso_gettimeofday = reinterpret_cast<decltype(&gettimeofday)>(
__libc_globals->vdso[VDSO_GETTIMEOFDAY].fn);
if (__predict_true(vdso_gettimeofday)) {
return vdso_return(vdso_gettimeofday(tv, tz));
}
return __gettimeofday(tv, tz);
}
time_t time(time_t* t) {
// Only x86/x86-64 actually have time() in the vdso.
#if defined(VDSO_TIME_SYMBOL)
auto vdso_time = reinterpret_cast<decltype(&time)>(__libc_globals->vdso[VDSO_TIME].fn);
if (__predict_true(vdso_time)) {
return vdso_time(t);
}
#endif
// We can't fallback to the time(2) system call because it doesn't exist for most architectures.
timeval tv;
if (gettimeofday(&tv, nullptr) == -1) return -1;
if (t) *t = tv.tv_sec;
return tv.tv_sec;
}
#if defined(__riscv)
int __riscv_hwprobe(struct riscv_hwprobe* _Nonnull pairs, size_t pair_count, size_t cpu_count,
unsigned long* _Nullable cpus, unsigned flags) {
auto vdso_riscv_hwprobe =
reinterpret_cast<decltype(&__riscv_hwprobe)>(__libc_globals->vdso[VDSO_RISCV_HWPROBE].fn);
if (__predict_true(vdso_riscv_hwprobe)) {
return -vdso_riscv_hwprobe(pairs, pair_count, cpu_count, cpus, flags);
}
// Inline the syscall directly in case someone's calling it from an
// ifunc resolver where we won't be able to set errno on failure.
// (Rather than our usual trick of letting the python-generated
// wrapper set errno but saving/restoring errno in cases where the API
// is to return an error value rather than setting errno.)
register long a0 __asm__("a0") = reinterpret_cast<long>(pairs);
register long a1 __asm__("a1") = pair_count;
register long a2 __asm__("a2") = cpu_count;
register long a3 __asm__("a3") = reinterpret_cast<long>(cpus);
register long a4 __asm__("a4") = flags;
register long a7 __asm__("a7") = __NR_riscv_hwprobe;
__asm__ volatile("ecall" : "=r"(a0) : "r"(a0), "r"(a1), "r"(a2), "r"(a3), "r"(a4), "r"(a7));
return -a0;
}
#endif
void __libc_init_vdso(libc_globals* globals) {
auto&& vdso = globals->vdso;
vdso[VDSO_CLOCK_GETTIME] = {VDSO_CLOCK_GETTIME_SYMBOL, nullptr};
vdso[VDSO_CLOCK_GETRES] = {VDSO_CLOCK_GETRES_SYMBOL, nullptr};
vdso[VDSO_GETTIMEOFDAY] = {VDSO_GETTIMEOFDAY_SYMBOL, nullptr};
#if defined(VDSO_TIME_SYMBOL)
vdso[VDSO_TIME] = {VDSO_TIME_SYMBOL, nullptr};
#endif
#if defined(VDSO_RISCV_HWPROBE_SYMBOL)
vdso[VDSO_RISCV_HWPROBE] = {VDSO_RISCV_HWPROBE_SYMBOL, nullptr};
#endif
// Do we have a vdso?
uintptr_t vdso_ehdr_addr = getauxval(AT_SYSINFO_EHDR);
ElfW(Ehdr)* vdso_ehdr = reinterpret_cast<ElfW(Ehdr)*>(vdso_ehdr_addr);
if (vdso_ehdr == nullptr) {
return;
}
// How many symbols does it have?
size_t symbol_count = 0;
ElfW(Shdr)* vdso_shdr = reinterpret_cast<ElfW(Shdr)*>(vdso_ehdr_addr + vdso_ehdr->e_shoff);
for (size_t i = 0; i < vdso_ehdr->e_shnum; ++i) {
if (vdso_shdr[i].sh_type == SHT_DYNSYM) {
symbol_count = vdso_shdr[i].sh_size / sizeof(ElfW(Sym));
break;
}
}
if (symbol_count == 0) {
return;
}
// Where's the dynamic table?
ElfW(Addr) vdso_addr = 0;
ElfW(Dyn)* vdso_dyn = nullptr;
ElfW(Phdr)* vdso_phdr = reinterpret_cast<ElfW(Phdr)*>(vdso_ehdr_addr + vdso_ehdr->e_phoff);
for (size_t i = 0; i < vdso_ehdr->e_phnum; ++i) {
if (vdso_phdr[i].p_type == PT_DYNAMIC) {
vdso_dyn = reinterpret_cast<ElfW(Dyn)*>(vdso_ehdr_addr + vdso_phdr[i].p_offset);
} else if (vdso_phdr[i].p_type == PT_LOAD) {
vdso_addr = vdso_ehdr_addr + vdso_phdr[i].p_offset - vdso_phdr[i].p_vaddr;
}
if (vdso_addr && vdso_dyn) break;
}
if (vdso_addr == 0 || vdso_dyn == nullptr) {
return;
}
// Where are the string and symbol tables?
const char* strtab = nullptr;
ElfW(Sym)* symtab = nullptr;
for (ElfW(Dyn)* d = vdso_dyn; d->d_tag != DT_NULL; ++d) {
if (d->d_tag == DT_STRTAB) {
strtab = reinterpret_cast<const char*>(vdso_addr + d->d_un.d_ptr);
} else if (d->d_tag == DT_SYMTAB) {
symtab = reinterpret_cast<ElfW(Sym)*>(vdso_addr + d->d_un.d_ptr);
}
if (strtab && symtab) break;
}
if (strtab == nullptr || symtab == nullptr) {
return;
}
// Are there any symbols we want?
for (size_t i = 0; i < VDSO_END; ++i) {
for (size_t j = 0; j < symbol_count; ++j) {
if (strcmp(vdso[i].name, strtab + symtab[j].st_name) == 0) {
vdso[i].fn = reinterpret_cast<void*>(vdso_addr + symtab[j].st_value);
break;
}
}
}
}
|
