bionic: add vdso time()

time() can be a hot call, and it currently uses __vdso_gettimeofday,
which is already pretty fast (~3 times faster than the syscall),
but with a __vdso_time call it is ~3 times even faster, in part
because __vdso_time does not require interlocking with updates,
and the read for just the seconds is atomic.  __vdso_time is
always available, whereas __vdso_gettimeofday is gated on access
to the physical timers.  arm improvement is compelling (x10),
x86 improvement is even more pronounced (x100).

[TL;DR]

w/vdso32 kernel patches, locked cores to MAX, little cores only.

BEFORE:

hikey960 vdso (aarch64):

----------------------------------------------------------------------
Benchmark                               Time           CPU Iterations
----------------------------------------------------------------------
BM_time_clock_gettime                  48 ns         48 ns   15414753
BM_time_clock_gettime_syscall         175 ns        175 ns    4062031
BM_time_clock_gettime_REALTIME         44 ns         44 ns   15897875
BM_time_clock_gettime_BOOTTIME         47 ns         47 ns   14307903
BM_time_clock_gettime_TAI             210 ns        210 ns    3341372
BM_time_clock_gettime_unsupported     100 ns        100 ns    7030649
BM_time_gettimeofday                   47 ns         47 ns   14969643
BM_time_gettimeofday_syscall          163 ns        163 ns    4283542
BM_time_time                           59 ns         59 ns   11815385

hikey960 vdso32 (aarch32):

----------------------------------------------------------------------
Benchmark                               Time           CPU Iterations
----------------------------------------------------------------------
BM_time_clock_gettime                  90 ns         90 ns    7572898
BM_time_clock_gettime_syscall         251 ns        251 ns    2763442
BM_time_clock_gettime_REALTIME         81 ns         80 ns    8699536
BM_time_clock_gettime_BOOTTIME         97 ns         97 ns    7256667
BM_time_clock_gettime_TAI             272 ns        272 ns    2570419
BM_time_clock_gettime_unsupported     160 ns        160 ns    4379819
BM_time_gettimeofday                   73 ns         73 ns    9608922
BM_time_gettimeofday_syscall          200 ns        199 ns    3527957
BM_time_time                          123 ns        123 ns    5651095

x86_64 (glibc):

--------------------------------------------------------------------
Benchmark                             Time           CPU Iterations
--------------------------------------------------------------------
BM_time_clock_gettime                  21 ns         21 ns   28873070
BM_time_clock_gettime_syscall         224 ns        224 ns    3095370
BM_time_clock_gettime_REALTIME         17 ns         17 ns   42083086
BM_time_clock_gettime_BOOTTIME        239 ns        239 ns    2924015
BM_time_clock_gettime_TAI             236 ns        236 ns    2961423
BM_time_clock_gettime_unsupported     221 ns        221 ns    3357696
BM_time_gettimeofday                 22 ns         22 ns   27975154
BM_time_gettimeofday_syscall        238 ns        238 ns    2882032
BM_time_time                          2 ns          2 ns  340354885
BM_time_time_syscall                207 ns        207 ns    3383073

imx7d_pico IOT nyc (w/arm,cpu-registers-not-fw-configured) (armv7a):
(virtual timers)

Benchmark                           Time(ns)    CPU(ns) Iterations
------------------------------------------------------------------
BM_time_clock_gettime                     20        477    1489362
BM_time_clock_gettime_syscall             20        487    1458333
BM_time_clock_gettime_REALTIME            19        464    1400000
BM_time_clock_gettime_BOOTTIME            29        700    1000000
BM_time_clock_gettime_TAI                 29        690    1000000
BM_time_clock_gettime_unsupported          9        227    3043478
BM_time_gettimeofday                      18        444    1555556
BM_time_gettimeofday_syscall              19        456    1555556
BM_time_time                              21        497    1166667

imx7d_pico IOT nyc (wo/arm,cpu-registers-not-fw-configured) (armv7a):
(physical timers)

Benchmark                           Time(ns)    CPU(ns) Iterations
------------------------------------------------------------------
BM_time_clock_gettime                      6        144    4666667
BM_time_clock_gettime_syscall             20        486    1400000
BM_time_clock_gettime_REALTIME             6        136    5000000
BM_time_clock_gettime_BOOTTIME             6        153    4375000
BM_time_clock_gettime_TAI                 31        760    1000000
BM_time_clock_gettime_unsupported         10        233    3043478
BM_time_gettimeofday                       6        140    5000000
BM_time_gettimeofday_syscall              19        450    1555556
BM_time_time                               9        203    3500000

AFTER:

hikey960 vdso (aarch64):

--------------------------------------------------------------------
Benchmark                             Time           CPU Iterations
--------------------------------------------------------------------
BM_time_clock_gettime                48 ns         48 ns   15414753
BM_time_clock_gettime_syscall       175 ns        175 ns    4062031
BM_time_clock_gettime_REALTIME       44 ns         44 ns   15897875
BM_time_clock_gettime_BOOTTIME       47 ns         47 ns   14307903
BM_time_clock_gettime_TAI           210 ns        210 ns    3341372
BM_time_clock_gettime_unsupported   100 ns        100 ns    7030649
BM_time_gettimeofday                 47 ns         47 ns   14975314
BM_time_gettimeofday_syscall        164 ns        164 ns    4278797
BM_time_time                         16 ns         16 ns   42932165

hikey960 vdso32 (aarch32):

--------------------------------------------------------------------
Benchmark                             Time           CPU Iterations
--------------------------------------------------------------------
BM_time_clock_gettime                90 ns         90 ns    7572898
BM_time_clock_gettime_syscall       251 ns        251 ns    2763442
BM_time_clock_gettime_REALTIME       81 ns         80 ns    8699536
BM_time_clock_gettime_BOOTTIME       97 ns         97 ns    7256667
BM_time_clock_gettime_TAI           272 ns        272 ns    2570419
BM_time_clock_gettime_unsupported   160 ns        160 ns    4379819
BM_time_gettimeofday                 73 ns         73 ns    9596230
BM_time_gettimeofday_syscall        199 ns        199 ns    3575428
BM_time_time                         35 ns         35 ns   19798801

imx7d_pico IOT nyc (w/arm,cpu-registers-not-fw-configured) (armv7a):

Benchmark                           Time(ns)    CPU(ns) Iterations
------------------------------------------------------------------
BM_time_clock_gettime                     20        477    1489362
BM_time_clock_gettime_syscall             20        487    1458333
BM_time_clock_gettime_REALTIME            19        464    1400000
BM_time_clock_gettime_BOOTTIME            29        700    1000000
BM_time_clock_gettime_TAI                 29        690    1000000
BM_time_clock_gettime_unsupported          9        227    3043478
BM_time_gettimeofday                      18        444    1555556
BM_time_gettimeofday_syscall              19        456    1555556
BM_time_time                               2         50   11666667

imx7d_pico IOT nyc (wo/arm,cpu-registers-not-fw-configured) (armv7a):

Benchmark                           Time(ns)    CPU(ns) Iterations
------------------------------------------------------------------
BM_time_clock_gettime                      6        144    4666667
BM_time_clock_gettime_syscall             20        486    1400000
BM_time_clock_gettime_REALTIME             6        136    5000000
BM_time_clock_gettime_BOOTTIME             6        153    4375000
BM_time_clock_gettime_TAI                 31        760    1000000
BM_time_clock_gettime_unsupported         10        233    3043478
BM_time_gettimeofday                       6        140    5000000
BM_time_gettimeofday_syscall              19        450    1555556
BM_time_time                               2         50   10000000

Test: bionic-unit-tests --gtest_filter=time.time
      taskset F bionic-benchmarks --bionic_xml=vdso.xml \
          --benchmark_filter='BM_time_(time*|clock_gettime*|gettimeofday*)'
Bug: 63737556
Change-Id: I81b088a12ca41a6c4733d46c5477527777138efa

3 files changed, 17 insertions, 3 deletions

diff --git a/libc/bionic/vdso.cpp b/libc/bionic/vdso.cpp
index 00d649039..44899e7cf 100644
--- a/libc/bionic/vdso.cpp
+++ b/libc/bionic/vdso.cpp
@@ -60,11 +60,21 @@ int gettimeofday(timeval* tv, struct timezone* tz) {
   return __gettimeofday(tv, tz);
 }
 
+time_t time(time_t* t) {
+  auto vdso_time = reinterpret_cast<decltype(&time)>(
+    __libc_globals->vdso[VDSO_TIME].fn);
+  if (__predict_true(vdso_time)) {
+    return vdso_time(t);
+  }
+  return __time(t);
+}
+
 void __libc_init_vdso(libc_globals* globals, KernelArgumentBlock& args) {
   auto&& vdso = globals->vdso;
   vdso[VDSO_CLOCK_GETTIME] = { VDSO_CLOCK_GETTIME_SYMBOL, nullptr };
   vdso[VDSO_CLOCK_GETRES] = { VDSO_CLOCK_GETRES_SYMBOL, nullptr };
   vdso[VDSO_GETTIMEOFDAY] = { VDSO_GETTIMEOFDAY_SYMBOL, nullptr };
+  vdso[VDSO_TIME] = { VDSO_TIME_SYMBOL, nullptr };
 
   // Do we have a vdso?
   uintptr_t vdso_ehdr_addr = args.getauxval(AT_SYSINFO_EHDR);
diff --git a/libc/private/bionic_vdso.h b/libc/private/bionic_vdso.h
index ed11501e1..8c4d8d2aa 100644
--- a/libc/private/bionic_vdso.h
+++ b/libc/private/bionic_vdso.h
@@ -35,15 +35,18 @@
 #define VDSO_CLOCK_GETTIME_SYMBOL "__kernel_clock_gettime"
 #define VDSO_CLOCK_GETRES_SYMBOL  "__kernel_clock_getres"
 #define VDSO_GETTIMEOFDAY_SYMBOL  "__kernel_gettimeofday"
+#define VDSO_TIME_SYMBOL          "__kernel_time"
 #else
 #define VDSO_CLOCK_GETTIME_SYMBOL "__vdso_clock_gettime"
 #define VDSO_CLOCK_GETRES_SYMBOL  "__vdso_clock_getres"
 #define VDSO_GETTIMEOFDAY_SYMBOL  "__vdso_gettimeofday"
+#define VDSO_TIME_SYMBOL          "__vdso_time"
 #endif
 
 extern "C" int __clock_gettime(int, timespec*);
 extern "C" int __clock_getres(int, timespec*);
 extern "C" int __gettimeofday(timeval*, struct timezone*);
+extern "C" time_t __time(time_t*);
 
 struct vdso_entry {
   const char* name;
@@ -54,6 +57,7 @@ enum {
   VDSO_CLOCK_GETTIME = 0,
   VDSO_CLOCK_GETRES,
   VDSO_GETTIMEOFDAY,
+  VDSO_TIME,
   VDSO_END
 };
 
diff --git a/libc/upstream-openbsd/lib/libc/gen/time.c b/libc/upstream-openbsd/lib/libc/gen/time.c
index 6fcf1cde2..afe463b64 100644
--- a/libc/upstream-openbsd/lib/libc/gen/time.c
+++ b/libc/upstream-openbsd/lib/libc/gen/time.c
@@ -28,11 +28,12 @@
  * SUCH DAMAGE.
  */
 
+#include <sys/cdefs.h>
 #include <sys/time.h>
 #include <time.h>
 
-time_t
-time(time_t *t)
+__LIBC_HIDDEN__ time_t
+__time(time_t *t)
 {
 	struct timeval tt;
 
@@ -42,4 +43,3 @@ time(time_t *t)
 		*t = (time_t)tt.tv_sec;
 	return (tt.tv_sec);
 }
-DEF_STRONG(time);