Source-Changes-HG archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
[src/trunk]: src/sys/dev/ic Fix the TSC timecounter (on the systems I have ac...
details: https://anonhg.NetBSD.org/src/rev/9642c955eb7d
branches: trunk
changeset: 971921:9642c955eb7d
user: ad <ad%NetBSD.org@localhost>
date: Fri May 08 22:01:54 2020 +0000
description:
Fix the TSC timecounter (on the systems I have access to):
- Make the early i8254-based calculation of frequency a bit more accurate.
- Keep track of how far the HPET & TSC advance between HPET attach and
secondary CPU boot, and use to compute an accurate value before attaching
the timecounter. Initial idea from joerg@.
- When determining skew and drift between CPUs, make each measurement 1000
times and pick the lowest observed value. Increase the error threshold to
1000 clock cycles.
- Use the frequency computed on the boot CPU for secondary CPUs too.
- Remove cpu_counter_serializing().
diffstat:
sys/arch/x86/include/cpu_counter.h | 3 +-
sys/arch/x86/x86/cpu.c | 76 +++++++++++++++++++++++++++-----------
sys/arch/x86/x86/tsc.c | 52 +++++++++++--------------
sys/dev/ic/hpet.c | 50 ++++++++++++++++++++++++-
sys/dev/ic/hpetvar.h | 3 +-
5 files changed, 128 insertions(+), 56 deletions(-)
diffs (truncated from 419 to 300 lines):
diff -r d79cad276c89 -r 9642c955eb7d sys/arch/x86/include/cpu_counter.h
--- a/sys/arch/x86/include/cpu_counter.h Fri May 08 21:58:03 2020 +0000
+++ b/sys/arch/x86/include/cpu_counter.h Fri May 08 22:01:54 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu_counter.h,v 1.5 2011/02/02 12:26:42 bouyer Exp $ */
+/* $NetBSD: cpu_counter.h,v 1.6 2020/05/08 22:01:54 ad Exp $ */
/*-
* Copyright (c) 2000, 2008 The NetBSD Foundation, Inc.
@@ -35,7 +35,6 @@
#ifdef _KERNEL
uint64_t cpu_counter(void);
-uint64_t cpu_counter_serializing(void);
uint32_t cpu_counter32(void);
uint64_t cpu_frequency(struct cpu_info *);
int cpu_hascounter(void);
diff -r d79cad276c89 -r 9642c955eb7d sys/arch/x86/x86/cpu.c
--- a/sys/arch/x86/x86/cpu.c Fri May 08 21:58:03 2020 +0000
+++ b/sys/arch/x86/x86/cpu.c Fri May 08 22:01:54 2020 +0000
@@ -1,7 +1,7 @@
-/* $NetBSD: cpu.c,v 1.189 2020/05/02 16:44:36 bouyer Exp $ */
+/* $NetBSD: cpu.c,v 1.190 2020/05/08 22:01:55 ad Exp $ */
/*
- * Copyright (c) 2000-2012 NetBSD Foundation, Inc.
+ * Copyright (c) 2000-2020 NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
@@ -62,7 +62,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.189 2020/05/02 16:44:36 bouyer Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.190 2020/05/08 22:01:55 ad Exp $");
#include "opt_ddb.h"
#include "opt_mpbios.h" /* for MPDEBUG */
@@ -73,6 +73,7 @@
#include "lapic.h"
#include "ioapic.h"
#include "acpica.h"
+#include "hpet.h"
#include <sys/param.h>
#include <sys/proc.h>
@@ -119,6 +120,7 @@
#endif
#include <dev/ic/mc146818reg.h>
+#include <dev/ic/hpetvar.h>
#include <i386/isa/nvram.h>
#include <dev/isa/isareg.h>
@@ -433,8 +435,14 @@
* must be done to allow booting other processors.
*/
if (!again) {
+ /* Make sure DELAY() (likely i8254_delay()) is initialized. */
+ DELAY(1);
+
+ /*
+ * Basic init. Compute an approximate frequency for the TSC
+ * using the i8254. If there's a HPET we'll redo it later.
+ */
atomic_or_32(&ci->ci_flags, CPUF_PRESENT | CPUF_PRIMARY);
- /* Basic init. */
cpu_intr_init(ci);
cpu_get_tsc_freq(ci);
cpu_init(ci);
@@ -451,8 +459,6 @@
lapic_calibrate_timer(ci);
}
#endif
- /* Make sure DELAY() is initialized. */
- DELAY(1);
kcsan_cpu_init(ci);
again = true;
}
@@ -718,7 +724,6 @@
if (ci != &cpu_info_primary) {
/* Synchronize TSC */
- wbinvd();
atomic_or_32(&ci->ci_flags, CPUF_RUNNING);
tsc_sync_ap(ci);
} else {
@@ -734,6 +739,14 @@
kcpuset_t *cpus;
u_long i;
+#if NHPET > 0
+ /* Use HPET delay, and re-calibrate TSC on boot CPU using HPET. */
+ if (hpet_delay_p() && x86_delay == i8254_delay) {
+ delay_func = x86_delay = hpet_delay;
+ cpu_get_tsc_freq(curcpu());
+ }
+#endif
+
/* Now that we know the number of CPUs, patch the text segment. */
x86_patch(false);
@@ -842,7 +855,6 @@
*/
psl = x86_read_psl();
x86_disable_intr();
- wbinvd();
tsc_sync_bp(ci);
x86_write_psl(psl);
}
@@ -873,7 +885,6 @@
drift = ci->ci_data.cpu_cc_skew;
psl = x86_read_psl();
x86_disable_intr();
- wbinvd();
tsc_sync_bp(ci);
x86_write_psl(psl);
drift -= ci->ci_data.cpu_cc_skew;
@@ -919,7 +930,6 @@
* Synchronize the TSC for the first time. Note that interrupts are
* off at this point.
*/
- wbinvd();
atomic_or_32(&ci->ci_flags, CPUF_PRESENT);
tsc_sync_ap(ci);
@@ -1310,21 +1320,43 @@
void
cpu_get_tsc_freq(struct cpu_info *ci)
{
- uint64_t freq = 0, last_tsc;
-
- if (cpu_hascounter())
- freq = cpu_tsc_freq_cpuid(ci);
+ uint64_t freq = 0, t0, t1;
+ int64_t overhead;
- if (freq != 0) {
- /* Use TSC frequency taken from CPUID. */
- ci->ci_data.cpu_cc_freq = freq;
+ if ((ci->ci_flags & CPUF_PRIMARY) != 0 && cpu_hascounter()) {
+ freq = cpu_tsc_freq_cpuid(ci);
+#if NHPET > 0
+ if (freq == 0)
+ freq = hpet_tsc_freq();
+#endif
+ if (freq == 0) {
+ /*
+ * Work out the approximate overhead involved below.
+ * Discard the result of the first go around the
+ * loop.
+ */
+ overhead = 0;
+ for (int i = 0; i <= 8; i++) {
+ t0 = cpu_counter();
+ x86_delay(0);
+ t1 = cpu_counter();
+ if (i > 0) {
+ overhead += (t1 - t0);
+ }
+ }
+ overhead >>= 3;
+
+ /* Now do the calibration. */
+ t0 = cpu_counter();
+ x86_delay(100000);
+ t1 = cpu_counter();
+ freq = (t1 - t0 - overhead) * 10;
+ }
} else {
- /* Calibrate TSC frequency. */
- last_tsc = cpu_counter_serializing();
- delay_func(100000);
- ci->ci_data.cpu_cc_freq =
- (cpu_counter_serializing() - last_tsc) * 10;
+ freq = cpu_info_primary.ci_data.cpu_cc_freq;
}
+
+ ci->ci_data.cpu_cc_freq = freq;
}
void
diff -r d79cad276c89 -r 9642c955eb7d sys/arch/x86/x86/tsc.c
--- a/sys/arch/x86/x86/tsc.c Fri May 08 21:58:03 2020 +0000
+++ b/sys/arch/x86/x86/tsc.c Fri May 08 22:01:54 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: tsc.c,v 1.43 2020/04/25 15:26:18 bouyer Exp $ */
+/* $NetBSD: tsc.c,v 1.44 2020/05/08 22:01:55 ad Exp $ */
/*-
* Copyright (c) 2008, 2020 The NetBSD Foundation, Inc.
@@ -27,7 +27,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: tsc.c,v 1.43 2020/04/25 15:26:18 bouyer Exp $");
+__KERNEL_RCSID(0, "$NetBSD: tsc.c,v 1.44 2020/05/08 22:01:55 ad Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -47,12 +47,14 @@
#include "tsc.h"
+#define TSC_SYNC_ROUNDS 1000
+#define ABS(a) ((a) >= 0 ? (a) : -(a))
+
u_int tsc_get_timecount(struct timecounter *);
uint64_t tsc_freq; /* exported for sysctl */
-static int64_t tsc_drift_max = 250; /* max cycles */
+static int64_t tsc_drift_max = 1000; /* max cycles */
static int64_t tsc_drift_observed;
-static bool tsc_good;
int tsc_user_enabled = 1;
@@ -158,9 +160,6 @@
ci = curcpu();
tsc_freq = ci->ci_data.cpu_cc_freq;
- tsc_good = (cpu_feature[0] & CPUID_MSR) != 0 &&
- (rdmsr(MSR_TSC) != 0 || rdmsr(MSR_TSC) != 0);
-
invariant = tsc_is_invariant();
if (!invariant) {
aprint_debug("TSC not known invariant on this CPU\n");
@@ -206,13 +205,12 @@
/* Flag it and read our TSC. */
atomic_or_uint(&ci->ci_flags, CPUF_SYNCTSC);
- bptsc = (rdtsc() >> 1);
/* Wait for remote to complete, and read ours again. */
while ((ci->ci_flags & CPUF_SYNCTSC) != 0) {
__insn_barrier();
}
- bptsc += (rdtsc() >> 1);
+ bptsc = rdtsc();
/* Wait for the results to come in. */
while (tsc_sync_cpu == ci) {
@@ -229,17 +227,21 @@
void
tsc_sync_bp(struct cpu_info *ci)
{
- int64_t bptsc, aptsc, bsum = 0, asum = 0;
+ int64_t bptsc, aptsc, val, diff;
+
+ if (!cpu_hascounter())
+ return;
- tsc_read_bp(ci, &bptsc, &aptsc); /* discarded - cache effects */
- for (int i = 0; i < 8; i++) {
+ val = INT64_MAX;
+ for (int i = 0; i < TSC_SYNC_ROUNDS; i++) {
tsc_read_bp(ci, &bptsc, &aptsc);
- bsum += bptsc;
- asum += aptsc;
+ diff = bptsc - aptsc;
+ if (ABS(diff) < ABS(val)) {
+ val = diff;
+ }
}
- /* Compute final value to adjust for skew. */
- ci->ci_data.cpu_cc_skew = (bsum - asum) >> 3;
+ ci->ci_data.cpu_cc_skew = val;
}
/*
@@ -255,11 +257,10 @@
while ((ci->ci_flags & CPUF_SYNCTSC) == 0) {
__insn_barrier();
}
- tsc = (rdtsc() >> 1);
/* Instruct primary to read its counter. */
atomic_and_uint(&ci->ci_flags, ~CPUF_SYNCTSC);
- tsc += (rdtsc() >> 1);
+ tsc = rdtsc();
/* Post result. Ensure the whole value goes out atomically. */
(void)atomic_swap_64(&tsc_sync_val, tsc);
@@ -273,8 +274,10 @@
tsc_sync_ap(struct cpu_info *ci)
{
- tsc_post_ap(ci);
- for (int i = 0; i < 8; i++) {
+ if (!cpu_hascounter())
+ return;
+
+ for (int i = 0; i < TSC_SYNC_ROUNDS; i++) {
tsc_post_ap(ci);
}
}
@@ -321,12 +324,3 @@
Home |
Main Index |
Thread Index |
Old Index