[src/trunk]: src/sys/sys kern: Make time_second and time_uptime macros that w...

[riastradh <riastradh%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/c231857432e9
branches:  trunk
changeset: 377557:c231857432e9
user:      riastradh <riastradh%NetBSD.org@localhost>
date:      Mon Jul 17 12:55:20 2023 +0000

description:
kern: Make time_second and time_uptime macros that work atomically.

These use atomic load on platforms with atomic 64-bit load, and
seqlocks on platforms without.

This has the unfortunate side effect of slightly reducing the real
times available on 32-bit platforms, from ending some time in the
year 584942417218 AD, available on 64-bit platforms, to ending some
time in the year 584942417355 AD.  But during that slightly shorter
time, 32-bit platforms can avoid bugs arising from non-atomic access
to time_uptime and time_second.

Note: All platforms still have non-atomic access problems for
bintime, binuptime, nanotime, nanouptime, &c.  This can be addressed
by putting a seqlock around timebasebin and possibly some other
variable -- to be done in a later change.

XXX kernel ABI change -- deleting symbols

diffstat:

 share/man/man9/time_second.9 |   34 ++++++++---
 sys/kern/kern_tc.c           |  120 ++++++++++++++++++++++++++++++++++++++++--
 sys/sys/timevar.h            |   50 ++++++++++++++++-
 3 files changed, 183 insertions(+), 21 deletions(-)

diffs (truncated from 314 to 300 lines):

diff -r 983df8579fa1 -r c231857432e9 share/man/man9/time_second.9
--- a/share/man/man9/time_second.9      Mon Jul 17 12:55:03 2023 +0000
+++ b/share/man/man9/time_second.9      Mon Jul 17 12:55:20 2023 +0000
@@ -1,4 +1,4 @@
-.\" $NetBSD: time_second.9,v 1.9 2020/04/18 18:55:20 wiz Exp $
+.\" $NetBSD: time_second.9,v 1.10 2023/07/17 12:55:20 riastradh Exp $
 .\"
 .\" Copyright (c) 1994 Christopher G. Demetriou
 .\" All rights reserved.
@@ -32,23 +32,27 @@
 .\"
 .\" <<Id: LICENSE,v 1.2 2000/06/14 15:57:33 cgd Exp>>
 .\"
-.Dd April 17, 2020
+.Dd July 16, 2023
 .Dt TIME_SECOND 9
 .Os
 .Sh NAME
 .Nm time_second ,
-.Nm time_uptime
+.Nm time_uptime ,
+.Nm time_uptime32
 .Nd system time variables
 .Sh SYNOPSIS
 .In sys/time.h
-.Vt extern time_t time_second;
-.Vt extern time_t time_uptime;
+.Vt extern const time_t time_second;
+.Vt extern const time_t time_uptime;
+.Vt extern const uint32_t time_uptime32;
 .Sh DESCRIPTION
 The
 .Va time_second
 variable is the system's
-.Dq wall time
-clock.
+.Dq wall clock ,
+giving the number of seconds since midnight (0 hour),
+January 1, 1970, (proleptic) UTC,
+minus the number of leap seconds.
 It is set at boot by
 .Xr inittodr 9 ,
 and is updated periodically via
@@ -64,9 +68,19 @@ It is set at boot, and is updated period
 (It is not updated by
 .Xr settimeofday 2 . )
 .Pp
-All of these variables contain times
-expressed in seconds since midnight (0 hour),
-January 1, 1970, UTC.
+The
+.Va time_uptime32
+variable is the low-order 32 bits of
+.Va time_uptime ,
+which is cheaper to read on 32-bit platforms.
+.Pp
+You must only read the variables
+.Va time_second ,
+.Va time_uptime ,
+and
+.Va time_uptime32 ;
+you may not write to them or take their addresses.
+They may be implemented as macros.
 .Pp
 The
 .Xr bintime 9 ,
diff -r 983df8579fa1 -r c231857432e9 sys/kern/kern_tc.c
--- a/sys/kern/kern_tc.c        Mon Jul 17 12:55:03 2023 +0000
+++ b/sys/kern/kern_tc.c        Mon Jul 17 12:55:20 2023 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: kern_tc.c,v 1.62 2021/06/02 21:34:58 riastradh Exp $ */
+/* $NetBSD: kern_tc.c,v 1.63 2023/07/17 12:55:20 riastradh Exp $ */
 
 /*-
  * Copyright (c) 2008, 2009 The NetBSD Foundation, Inc.
@@ -40,17 +40,19 @@
 
 #include <sys/cdefs.h>
 /* __FBSDID("$FreeBSD: src/sys/kern/kern_tc.c,v 1.166 2005/09/19 22:16:31 andre Exp $"); */
-__KERNEL_RCSID(0, "$NetBSD: kern_tc.c,v 1.62 2021/06/02 21:34:58 riastradh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_tc.c,v 1.63 2023/07/17 12:55:20 riastradh Exp $");
 
 #ifdef _KERNEL_OPT
 #include "opt_ntp.h"
 #endif
 
 #include <sys/param.h>
+
 #include <sys/atomic.h>
 #include <sys/evcnt.h>
 #include <sys/kauth.h>
 #include <sys/kernel.h>
+#include <sys/lock.h>
 #include <sys/mutex.h>
 #include <sys/reboot.h>        /* XXX just to get AB_VERBOSE */
 #include <sys/sysctl.h>
@@ -131,8 +133,18 @@ static struct timehands *volatile timeha
 struct timecounter *timecounter = &dummy_timecounter;
 static struct timecounter *timecounters = &dummy_timecounter;
 
-volatile time_t time_second __cacheline_aligned = 1;
-volatile time_t time_uptime __cacheline_aligned = 1;
+volatile time_t time__second __cacheline_aligned = 1;
+volatile time_t time__uptime __cacheline_aligned = 1;
+
+#ifndef __HAVE_ATOMIC64_LOADSTORE
+static volatile struct {
+       uint32_t lo, hi;
+} time__uptime32 __cacheline_aligned = {
+       .lo = 1,
+}, time__second32 __cacheline_aligned = {
+       .lo = 1,
+};
+#endif
 
 static struct bintime timebasebin;
 
@@ -143,6 +155,103 @@ static u_int timecounter_mods;
 static volatile int timecounter_removals = 1;
 static u_int timecounter_bad;
 
+#ifdef __HAVE_ATOMIC64_LOADSTORE
+
+static inline void
+setrealuptime(time_t second, time_t uptime)
+{
+
+       atomic_store_relaxed(&time__second, second);
+       atomic_store_relaxed(&time__uptime, uptime);
+}
+
+#else
+
+static inline void
+setrealuptime(time_t second, time_t uptime)
+{
+       uint32_t seclo = second & 0xffffffff, sechi = second >> 32;
+       uint32_t uplo = uptime & 0xffffffff, uphi = uptime >> 32;
+
+       KDASSERT(mutex_owned(&timecounter_lock));
+
+       /*
+        * Fast path -- no wraparound, just updating the low bits, so
+        * no need for seqlocked access.
+        */
+       if (__predict_true(sechi == time__second32.hi) &&
+           __predict_true(uphi == time__uptime32.hi)) {
+               atomic_store_relaxed(&time__second32.lo, seclo);
+               atomic_store_relaxed(&time__uptime32.lo, uplo);
+               return;
+       }
+
+       atomic_store_relaxed(&time__second32.hi, 0xffffffff);
+       atomic_store_relaxed(&time__uptime32.hi, 0xffffffff);
+       membar_producer();
+       atomic_store_relaxed(&time__second32.lo, seclo);
+       atomic_store_relaxed(&time__uptime32.lo, uplo);
+       membar_producer();
+       atomic_store_relaxed(&time__second32.hi, sechi);
+       atomic_store_relaxed(&time__second32.lo, seclo);
+}
+
+time_t
+getrealtime(void)
+{
+       uint32_t lo, hi;
+
+       do {
+               for (;;) {
+                       hi = atomic_load_relaxed(&time__second32.hi);
+                       if (__predict_true(hi != 0xffffffff))
+                               break;
+                       SPINLOCK_BACKOFF_HOOK;
+               }
+               membar_consumer();
+               lo = atomic_load_relaxed(&time__second32.lo);
+               membar_consumer();
+       } while (hi != atomic_load_relaxed(&time__second32.hi));
+
+       return ((time_t)hi << 32) | lo;
+}
+
+time_t
+getuptime(void)
+{
+       uint32_t lo, hi;
+
+       do {
+               for (;;) {
+                       hi = atomic_load_relaxed(&time__uptime32.hi);
+                       if (__predict_true(hi != 0xffffffff))
+                               break;
+                       SPINLOCK_BACKOFF_HOOK;
+               }
+               membar_consumer();
+               lo = atomic_load_relaxed(&time__uptime32.lo);
+               membar_consumer();
+       } while (hi != atomic_load_relaxed(&time__uptime32.hi));
+
+       return ((time_t)hi << 32) | lo;
+}
+
+time_t
+getboottime(void)
+{
+
+       return getrealtime() - getuptime();
+}
+
+uint32_t
+getuptime32(void)
+{
+
+       return atomic_load_relaxed(&time__uptime32.lo);
+}
+
+#endif /* !defined(__HAVE_ATOMIC64_LOADSTORE) */
+
 /*
  * sysctl helper routine for kern.timercounter.hardware
  */
@@ -878,8 +987,7 @@ tc_windup(void)
         * Go live with the new struct timehands.  Ensure changes are
         * globally visible before changing.
         */
-       time_second = th->th_microtime.tv_sec;
-       time_uptime = th->th_offset.sec;
+       setrealuptime(th->th_microtime.tv_sec, th->th_offset.sec);
        membar_producer();
        timehands = th;
 
diff -r 983df8579fa1 -r c231857432e9 sys/sys/timevar.h
--- a/sys/sys/timevar.h Mon Jul 17 12:55:03 2023 +0000
+++ b/sys/sys/timevar.h Mon Jul 17 12:55:20 2023 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: timevar.h,v 1.49 2022/10/26 23:23:52 riastradh Exp $   */
+/*     $NetBSD: timevar.h,v 1.50 2023/07/17 12:55:20 riastradh Exp $   */
 
 /*
  *  Copyright (c) 2005, 2008, 2020 The NetBSD Foundation, Inc.
@@ -60,6 +60,7 @@
 #ifndef _SYS_TIMEVAR_H_
 #define _SYS_TIMEVAR_H_
 
+#include <sys/atomic.h>
 #include <sys/callout.h>
 #include <sys/queue.h>
 #include <sys/signal.h>
@@ -234,8 +235,47 @@ void       itimer_gettime(const struct itimer 
 void   ptimer_tick(struct lwp *, bool);
 void   ptimers_free(struct proc *, int);
 
-extern volatile time_t time_second;    /* current second in the epoch */
-extern volatile time_t time_uptime;    /* system uptime in seconds */
+extern volatile time_t time__second;   /* current second in the epoch */
+extern volatile time_t time__uptime;   /* system uptime in seconds */
+
+#define        time_second     getrealtime()
+#define        time_uptime     getuptime()
+#define        time_uptime32   getuptime32()
+
+#ifdef __HAVE_ATOMIC64_LOADSTORE
+
+static inline time_t
+getrealtime(void)
+{
+       return atomic_load_relaxed(&time__second);
+}
+
+static inline time_t
+getuptime(void)
+{
+       return atomic_load_relaxed(&time__uptime);
+}
+
+static inline time_t
+getboottime(void)
+{
+       return getrealtime() - getuptime();
+}
+
+static inline uint32_t
+getuptime32(void)
+{
+       return getuptime() & 0xffffffff;
+}
+
+#else
+
+time_t         getrealtime(void);
+time_t         getuptime(void);
+time_t         getboottime(void);
+uint32_t       getuptime32(void);
+
+#endif
 
 extern int time_adjusted;
 
@@ -248,13 +288,13 @@ extern int time_adjusted;
 static __inline time_t time_mono_to_wall(time_t t)