[src/trunk]: src/sys/arch/x86/x86 The boot CPU suffers a cache miss during TS...

[ad <ad%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/6539a9d68bce
branches:  trunk
changeset: 933174:6539a9d68bce
user:      ad <ad%NetBSD.org@localhost>
date:      Wed May 20 20:19:02 2020 +0000

description:
The boot CPU suffers a cache miss during TSC sync, before RDTSC.  Make the
secondary CPU take a miss as well to try and delay it an equal amount.

diffstat:

 sys/arch/x86/x86/tsc.c |  23 ++++++++++++++++-------
 1 files changed, 16 insertions(+), 7 deletions(-)

diffs (74 lines):

diff -r 3e22cf34798a -r 6539a9d68bce sys/arch/x86/x86/tsc.c
--- a/sys/arch/x86/x86/tsc.c    Wed May 20 20:18:37 2020 +0000
+++ b/sys/arch/x86/x86/tsc.c    Wed May 20 20:19:02 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: tsc.c,v 1.46 2020/05/19 21:56:51 ad Exp $      */
+/*     $NetBSD: tsc.c,v 1.47 2020/05/20 20:19:02 ad Exp $      */
 
 /*-
  * Copyright (c) 2008, 2020 The NetBSD Foundation, Inc.
@@ -27,7 +27,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: tsc.c,v 1.46 2020/05/19 21:56:51 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: tsc.c,v 1.47 2020/05/20 20:19:02 ad Exp $");
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -54,6 +54,7 @@
 
 static void    tsc_delay(unsigned int);
 
+static uint64_t        tsc_dummy_cacheline __cacheline_aligned;
 uint64_t       tsc_freq; /* exported for sysctl */
 static int64_t tsc_drift_max = 1000;   /* max cycles */
 static int64_t tsc_drift_observed;
@@ -200,7 +201,7 @@
  * Called during startup of APs, by the boot processor.  Interrupts
  * are disabled on entry.
  */
-static void
+static void __noinline
 tsc_read_bp(struct cpu_info *ci, uint64_t *bptscp, uint64_t *aptscp)
 {
        uint64_t bptsc;
@@ -209,10 +210,13 @@
                panic("tsc_sync_bp: 1");
        }
 
-       /* Flag it and read our TSC. */
+       /* Prepare a cache miss for the other side. */
+       (void)atomic_swap_uint((void *)&tsc_dummy_cacheline, 0);
+
+       /* Flag our readiness. */
        atomic_or_uint(&ci->ci_flags, CPUF_SYNCTSC);
 
-       /* Wait for remote to complete, and read ours again. */
+       /* Wait for other side then read our TSC. */
        while ((ci->ci_flags & CPUF_SYNCTSC) != 0) {
                __insn_barrier();
        }
@@ -254,7 +258,7 @@
  * Called during startup of AP, by the AP itself.  Interrupts are
  * disabled on entry.
  */
-static void
+static void __noinline
 tsc_post_ap(struct cpu_info *ci)
 {
        uint64_t tsc;
@@ -266,7 +270,12 @@
 
        /* Instruct primary to read its counter. */
        atomic_and_uint(&ci->ci_flags, ~CPUF_SYNCTSC);
-       tsc = rdtsc();
+
+       /* Suffer a cache miss, then read TSC. */
+       __insn_barrier();
+       tsc = tsc_dummy_cacheline;
+       __insn_barrier();
+       tsc += rdtsc();
 
        /* Post result.  Ensure the whole value goes out atomically. */
        (void)atomic_swap_64(&tsc_sync_val, tsc);