> Date: Sun, 6 Nov 2022 19:42:11 +0000
> From: Scott Cheloha <[email protected]>
>
> This patch switches arm64 to clockintr(9).
>
> This has survived about a dozen parallel kernel builds, parallel
> release builds, and bsd.rd upgrades on my RPi4b.
>
> kettenis@ said he tried it on one of his M1 machines, too.
>
> It could use more testing, particularly on arm64 machines that aren't
> a Raspberry Pi.
>
> Notes:
>
> - The spec is a little unclear about whether or not the
> values written to CNTV_TVAL_EL0 are interpretted as signed
> or unsigned. To be safe, I have capped the cycle count at
> INT32_MAX.
ok kettenis@
> Index: sys/arch/arm64/dev/agtimer.c
> ===================================================================
> RCS file: /cvs/src/sys/arch/arm64/dev/agtimer.c,v
> retrieving revision 1.19
> diff -u -p -r1.19 agtimer.c
> --- sys/arch/arm64/dev/agtimer.c 24 Oct 2021 17:52:28 -0000 1.19
> +++ sys/arch/arm64/dev/agtimer.c 6 Nov 2022 18:48:22 -0000
> @@ -18,10 +18,12 @@
>
> #include <sys/param.h>
> #include <sys/systm.h>
> +#include <sys/clockintr.h>
> #include <sys/queue.h>
> #include <sys/malloc.h>
> #include <sys/device.h>
> #include <sys/kernel.h>
> +#include <sys/stdint.h>
> #include <sys/timetc.h>
> #include <sys/evcount.h>
>
> @@ -53,28 +55,13 @@ static struct timecounter agtimer_timeco
> .tc_user = TC_AGTIMER,
> };
>
> -struct agtimer_pcpu_softc {
> - uint64_t pc_nexttickevent;
> - uint64_t pc_nextstatevent;
> - u_int32_t pc_ticks_err_sum;
> -};
> -
> struct agtimer_softc {
> struct device sc_dev;
> int sc_node;
>
> - struct agtimer_pcpu_softc sc_pstat[MAXCPUS];
> -
> - u_int32_t sc_ticks_err_cnt;
> u_int32_t sc_ticks_per_second;
> - u_int32_t sc_ticks_per_intr;
> - u_int32_t sc_statvar;
> - u_int32_t sc_statmin;
> -
> -#ifdef AMPTIMER_DEBUG
> - struct evcount sc_clk_count;
> - struct evcount sc_stat_count;
> -#endif
> + uint64_t sc_nsec_cycle_ratio;
> + uint64_t sc_nsec_max;
> void *sc_ih;
> };
>
> @@ -84,9 +71,11 @@ uint64_t agtimer_readcnt64(void);
> int agtimer_intr(void *);
> void agtimer_cpu_initclocks(void);
> void agtimer_delay(u_int);
> +void agtimer_rearm(void *, uint64_t);
> void agtimer_setstatclockrate(int stathz);
> void agtimer_set_clockrate(int32_t new_frequency);
> void agtimer_startclock(void);
> +void agtimer_trigger(void *);
>
> const struct cfattach agtimer_ca = {
> sizeof (struct agtimer_softc), agtimer_match, agtimer_attach
> @@ -96,6 +85,11 @@ struct cfdriver agtimer_cd = {
> NULL, "agtimer", DV_DULL
> };
>
> +struct intrclock agtimer_intrclock = {
> + .ic_rearm = agtimer_rearm,
> + .ic_trigger = agtimer_trigger
> +};
> +
> uint64_t
> agtimer_readcnt64(void)
> {
> @@ -173,13 +167,11 @@ agtimer_attach(struct device *parent, st
> agtimer_frequency =
> OF_getpropint(sc->sc_node, "clock-frequency", agtimer_frequency);
> sc->sc_ticks_per_second = agtimer_frequency;
> + sc->sc_nsec_cycle_ratio =
> + sc->sc_ticks_per_second * (1ULL << 32) / 1000000000;
> + sc->sc_nsec_max = UINT64_MAX / sc->sc_nsec_cycle_ratio;
>
> - printf(": %d kHz\n", sc->sc_ticks_per_second / 1000);
> -
> -#ifdef AMPTIMER_DEBUG
> - evcount_attach(&sc->sc_clk_count, "clock", NULL);
> - evcount_attach(&sc->sc_stat_count, "stat", NULL);
> -#endif
> + printf(": %u kHz\n", sc->sc_ticks_per_second / 1000);
>
> /*
> * private timer and interrupts not enabled until
> @@ -191,8 +183,9 @@ agtimer_attach(struct device *parent, st
>
> agtimer_timecounter.tc_frequency = sc->sc_ticks_per_second;
> agtimer_timecounter.tc_priv = sc;
> -
> tc_init(&agtimer_timecounter);
> +
> + agtimer_intrclock.ic_cookie = sc;
> }
>
> u_int
> @@ -209,72 +202,30 @@ agtimer_get_timecount(struct timecounter
> return (val & 0xffffffff);
> }
>
> -int
> -agtimer_intr(void *frame)
> +void
> +agtimer_rearm(void *cookie, uint64_t nsecs)
> {
> - struct agtimer_softc *sc = agtimer_cd.cd_devs[0];
> - struct agtimer_pcpu_softc *pc = &sc->sc_pstat[CPU_INFO_UNIT(curcpu())];
> - uint64_t now;
> - uint64_t nextevent;
> - uint32_t r;
> -#if defined(USE_GTIMER_CMP)
> - int skip = 1;
> -#else
> - int64_t delay;
> -#endif
> - int rc = 0;
> -
> - /*
> - * DSR - I know that the tick timer is 64 bits, but the following
> - * code deals with rollover, so there is no point in dealing
> - * with the 64 bit math, just let the 32 bit rollover
> - * do the right thing
> - */
> -
> - now = agtimer_readcnt64();
> + struct agtimer_softc *sc = cookie;
> + uint32_t cycles;
>
> - while (pc->pc_nexttickevent <= now) {
> - pc->pc_nexttickevent += sc->sc_ticks_per_intr;
> - pc->pc_ticks_err_sum += sc->sc_ticks_err_cnt;
> -
> - /* looping a few times is faster than divide */
> - while (pc->pc_ticks_err_sum > hz) {
> - pc->pc_nexttickevent += 1;
> - pc->pc_ticks_err_sum -= hz;
> - }
> -
> -#ifdef AMPTIMER_DEBUG
> - sc->sc_clk_count.ec_count++;
> -#endif
> - rc = 1;
> - hardclock(frame);
> - }
> - while (pc->pc_nextstatevent <= now) {
> - do {
> - r = random() & (sc->sc_statvar -1);
> - } while (r == 0); /* random == 0 not allowed */
> - pc->pc_nextstatevent += sc->sc_statmin + r;
> -
> - /* XXX - correct nextstatevent? */
> -#ifdef AMPTIMER_DEBUG
> - sc->sc_stat_count.ec_count++;
> -#endif
> - rc = 1;
> - statclock(frame);
> - }
> -
> - if (pc->pc_nexttickevent < pc->pc_nextstatevent)
> - nextevent = pc->pc_nexttickevent;
> - else
> - nextevent = pc->pc_nextstatevent;
> -
> - delay = nextevent - now;
> - if (delay < 0)
> - delay = 1;
> + if (nsecs > sc->sc_nsec_max)
> + nsecs = sc->sc_nsec_max;
> + cycles = (nsecs * sc->sc_nsec_cycle_ratio) >> 32;
> + if (cycles > INT32_MAX)
> + cycles = INT32_MAX;
> + agtimer_set_tval(cycles);
> +}
>
> - agtimer_set_tval(delay);
> +void
> +agtimer_trigger(void *unused)
> +{
> + agtimer_set_tval(0);
> +}
>
> - return (rc);
> +int
> +agtimer_intr(void *frame)
> +{
> + return clockintr_dispatch(frame);
> }
>
> void
> @@ -288,8 +239,13 @@ agtimer_set_clockrate(int32_t new_freque
> return;
>
> sc->sc_ticks_per_second = agtimer_frequency;
> + sc->sc_nsec_cycle_ratio =
> + sc->sc_ticks_per_second * (1ULL << 32) / 1000000000;
> + sc->sc_nsec_max = UINT64_MAX / sc->sc_nsec_cycle_ratio;
> +
> agtimer_timecounter.tc_frequency = sc->sc_ticks_per_second;
> - printf("agtimer0: adjusting clock: new tick rate %d kHz\n",
> +
> + printf("agtimer0: adjusting clock: new tick rate %u kHz\n",
> sc->sc_ticks_per_second / 1000);
> }
>
> @@ -297,37 +253,31 @@ void
> agtimer_cpu_initclocks(void)
> {
> struct agtimer_softc *sc = agtimer_cd.cd_devs[0];
> - struct agtimer_pcpu_softc *pc = &sc->sc_pstat[CPU_INFO_UNIT(curcpu())];
> uint32_t reg;
> - uint64_t next;
> uint64_t kctl;
>
> stathz = hz;
> - profhz = hz * 10;
> + profhz = stathz * 10;
> + clockintr_init(CL_RNDSTAT);
>
> if (sc->sc_ticks_per_second != agtimer_frequency) {
> agtimer_set_clockrate(agtimer_frequency);
> }
>
> - agtimer_setstatclockrate(stathz);
> -
> - sc->sc_ticks_per_intr = sc->sc_ticks_per_second / hz;
> - sc->sc_ticks_err_cnt = sc->sc_ticks_per_second % hz;
> - pc->pc_ticks_err_sum = 0;
> -
> /* configure virtual timer interrupt */
> sc->sc_ih = arm_intr_establish_fdt_idx(sc->sc_node, 2,
> IPL_CLOCK|IPL_MPSAFE, agtimer_intr, NULL, "tick");
>
> - next = agtimer_readcnt64() + sc->sc_ticks_per_intr;
> - pc->pc_nexttickevent = pc->pc_nextstatevent = next;
> + clockintr_cpu_init(&agtimer_intrclock);
>
> reg = agtimer_get_ctrl();
> reg &= ~GTIMER_CNTV_CTL_IMASK;
> reg |= GTIMER_CNTV_CTL_ENABLE;
> - agtimer_set_tval(sc->sc_ticks_per_second);
> + agtimer_set_tval(INT32_MAX);
> agtimer_set_ctrl(reg);
>
> + clockintr_trigger();
> +
> /* enable userland access to virtual counter */
> kctl = READ_SPECIALREG(CNTKCTL_EL1);
> WRITE_SPECIALREG(CNTKCTL_EL1, kctl | CNTKCTL_EL0VCTEN);
> @@ -367,48 +317,27 @@ agtimer_delay(u_int usecs)
> void
> agtimer_setstatclockrate(int newhz)
> {
> - struct agtimer_softc *sc = agtimer_cd.cd_devs[0];
> - int minint, statint;
> - int s;
> -
> - s = splclock();
> -
> - statint = sc->sc_ticks_per_second / newhz;
> - /* calculate largest 2^n which is smaller that just over half statint */
> - sc->sc_statvar = 0x40000000; /* really big power of two */
> - minint = statint / 2 + 100;
> - while (sc->sc_statvar > minint)
> - sc->sc_statvar >>= 1;
> -
> - sc->sc_statmin = statint - (sc->sc_statvar >> 1);
> -
> - splx(s);
> -
> - /*
> - * XXX this allows the next stat timer to occur then it switches
> - * to the new frequency. Rather than switching instantly.
> - */
> + clockintr_setstatclockrate(newhz);
> }
>
> void
> agtimer_startclock(void)
> {
> struct agtimer_softc *sc = agtimer_cd.cd_devs[0];
> - struct agtimer_pcpu_softc *pc = &sc->sc_pstat[CPU_INFO_UNIT(curcpu())];
> - uint64_t nextevent;
> uint64_t kctl;
> uint32_t reg;
>
> - nextevent = agtimer_readcnt64() + sc->sc_ticks_per_intr;
> - pc->pc_nexttickevent = pc->pc_nextstatevent = nextevent;
> -
> arm_intr_route(sc->sc_ih, 1, curcpu());
>
> + clockintr_cpu_init(&agtimer_intrclock);
> +
> reg = agtimer_get_ctrl();
> reg &= ~GTIMER_CNTV_CTL_IMASK;
> reg |= GTIMER_CNTV_CTL_ENABLE;
> - agtimer_set_tval(sc->sc_ticks_per_second);
> + agtimer_set_tval(INT32_MAX);
> agtimer_set_ctrl(reg);
> +
> + clockintr_trigger();
>
> /* enable userland access to virtual counter */
> kctl = READ_SPECIALREG(CNTKCTL_EL1);
> Index: sys/arch/arm64/include/cpu.h
> ===================================================================
> RCS file: /cvs/src/sys/arch/arm64/include/cpu.h,v
> retrieving revision 1.28
> diff -u -p -r1.28 cpu.h
> --- sys/arch/arm64/include/cpu.h 29 Aug 2022 02:01:18 -0000 1.28
> +++ sys/arch/arm64/include/cpu.h 6 Nov 2022 18:48:22 -0000
> @@ -81,6 +81,7 @@ void arm32_vector_init(vaddr_t, int);
> * Per-CPU information. For now we assume one CPU.
> */
>
> +#include <sys/clockintr.h>
> #include <sys/device.h>
> #include <sys/sched.h>
> #include <sys/srp.h>
> @@ -142,7 +143,7 @@ struct cpu_info {
> #ifdef GPROF
> struct gmonparam *ci_gmon;
> #endif
> -
> + struct clockintr_queue ci_queue;
> char ci_panicbuf[512];
> };
>
> Index: sys/arch/arm64/include/_types.h
> ===================================================================
> RCS file: /cvs/src/sys/arch/arm64/include/_types.h,v
> retrieving revision 1.4
> diff -u -p -r1.4 _types.h
> --- sys/arch/arm64/include/_types.h 5 Mar 2018 01:15:25 -0000 1.4
> +++ sys/arch/arm64/include/_types.h 6 Nov 2022 18:48:22 -0000
> @@ -34,6 +34,8 @@
> #ifndef _MACHINE__TYPES_H_
> #define _MACHINE__TYPES_H_
>
> +#define __HAVE_CLOCKINTR
> +
> #if defined(_KERNEL)
> typedef struct label_t {
> long val[13];
>
>
