Take an effort to recode the arm64 vdso code from assembler to C
previously submitted by Andrew Pinski <apin...@cavium.com>, rework
it for use in both arm and arm64, overlapping any optimizations
for each architecture. But instead of landing it in arm64, land the
result into lib/vdso and unify both implementations to simplify
future maintenance.
Rename seq_count to tb_seq_count. Rename tk_is_cntvct to use_syscall.
Rename cs_mult to cs_mono_mult. All to align with the variables in the
arm64 vdso datapage. Rework vdso_read_begin() and vdso_read_retry()
functions to reflect modern access patterns for tb_seq_count field.
Update copyright message to reflect the start of the contributions in
this series.
Signed-off-by: Mark Salyzyn <saly...@android.com>
Tested-by: Mark Salyzyn <saly...@android.com>
Cc: James Morse <james.mo...@arm.com>
Cc: Russell King <li...@armlinux.org.uk>
Cc: Catalin Marinas <catalin.mari...@arm.com>
Cc: Will Deacon <will.dea...@arm.com>
Cc: Andy Lutomirski <l...@amacapital.net>
Cc: Dmitry Safonov <dsafo...@virtuozzo.com>
Cc: John Stultz <john.stu...@linaro.org>
Cc: Mark Rutland <mark.rutl...@arm.com>
Cc: Laura Abbott <labb...@redhat.com>
Cc: Kees Cook <keesc...@chromium.org>
Cc: Ard Biesheuvel <ard.biesheu...@linaro.org>
Cc: Andy Gross <andy.gr...@linaro.org>
Cc: Kevin Brodsky <kevin.brod...@arm.com>
Cc: Andrew Pinski <apin...@cavium.com>
Cc: Thomas Gleixner <t...@linutronix.de>
Cc: linux-kernel@vger.kernel.org
Cc: linux-arm-ker...@lists.infradead.org
Cc: Jeremy Linton <jeremy.lin...@arm.com>
Cc: android-ker...@android.com
v2:
- split first CL into 1 of 7 pieces
v4:
- update commit message to reflect reasoning
v5:
- rebase
---
arch/arm/include/asm/vdso_datapage.h | 6 +--
arch/arm/kernel/vdso.c | 17 ++++----
arch/arm/vdso/vgettimeofday.c | 61 +++++++++++++++-------------
3 files changed, 45 insertions(+), 39 deletions(-)
diff --git a/arch/arm/include/asm/vdso_datapage.h
b/arch/arm/include/asm/vdso_datapage.h
index 9be259442fca..fa3e1856244d 100644
--- a/arch/arm/include/asm/vdso_datapage.h
+++ b/arch/arm/include/asm/vdso_datapage.h
@@ -29,8 +29,8 @@
* 32 bytes.
*/
struct vdso_data {
- u32 seq_count; /* sequence count - odd during updates */
- u16 tk_is_cntvct; /* fall back to syscall if false */
+ u32 tb_seq_count; /* sequence count - odd during updates */
+ u16 use_syscall; /* fall back to syscall if true */
u16 cs_shift; /* clocksource shift */
u32 xtime_coarse_sec; /* coarse time */
u32 xtime_coarse_nsec;
@@ -38,7 +38,7 @@ struct vdso_data {
u32 wtm_clock_sec; /* wall to monotonic offset */
u32 wtm_clock_nsec;
u32 xtime_clock_sec; /* CLOCK_REALTIME - seconds */
- u32 cs_mult; /* clocksource multiplier */
+ u32 cs_mono_mult; /* clocksource multiplier */
u64 cs_cycle_last; /* last cycle value */
u64 cs_mask; /* clocksource mask */
diff --git a/arch/arm/kernel/vdso.c b/arch/arm/kernel/vdso.c
index f4dd7f9663c1..c2c57f6b8c60 100644
--- a/arch/arm/kernel/vdso.c
+++ b/arch/arm/kernel/vdso.c
@@ -276,14 +276,14 @@ void arm_install_vdso(struct mm_struct *mm, unsigned long
addr)
static void vdso_write_begin(struct vdso_data *vdata)
{
- ++vdso_data->seq_count;
+ ++vdso_data->tb_seq_count;
smp_wmb(); /* Pairs with smp_rmb in vdso_read_retry */
}
static void vdso_write_end(struct vdso_data *vdata)
{
smp_wmb(); /* Pairs with smp_rmb in vdso_read_begin */
- ++vdso_data->seq_count;
+ ++vdso_data->tb_seq_count;
}
static bool tk_is_cntvct(const struct timekeeper *tk)
@@ -307,10 +307,10 @@ static bool tk_is_cntvct(const struct timekeeper *tk)
* counter again, making it even, indicating to userspace that the
* update is finished.
*
- * Userspace is expected to sample seq_count before reading any other
- * fields from the data page. If seq_count is odd, userspace is
+ * Userspace is expected to sample tb_seq_count before reading any other
+ * fields from the data page. If tb_seq_count is odd, userspace is
* expected to wait until it becomes even. After copying data from
- * the page, userspace must sample seq_count again; if it has changed
+ * the page, userspace must sample tb_seq_count again; if it has changed
* from its previous value, userspace must retry the whole sequence.
*
* Calls to update_vsyscall are serialized by the timekeeping core.
@@ -328,18 +328,19 @@ void update_vsyscall(struct timekeeper *tk)
vdso_write_begin(vdso_data);
- vdso_data->tk_is_cntvct = tk_is_cntvct(tk);
+ vdso_data->use_syscall = !tk_is_cntvct(tk);
vdso_data->xtime_coarse_sec = tk->xtime_sec;
vdso_data->xtime_coarse_nsec = (u32)(tk->tkr_mono.xtime_nsec
>>
tk->tkr_mono.shift);
vdso_data->wtm_clock_sec = wtm->tv_sec;
vdso_data->wtm_clock_nsec = wtm->tv_nsec;
- if (vdso_data->tk_is_cntvct) {
+ if (!vdso_data->use_syscall) {
vdso_data->cs_cycle_last = tk->tkr_mono.cycle_last;
vdso_data->xtime_clock_sec = tk->xtime_sec;
vdso_data->xtime_clock_snsec = tk->tkr_mono.xtime_nsec;
- vdso_data->cs_mult = tk->tkr_mono.mult;
+ vdso_data->cs_mono_mult = tk->tkr_mono.mult;
+ /* tkr_mono.shift == tkr_raw.shift */
vdso_data->cs_shift = tk->tkr_mono.shift;
vdso_data->cs_mask = tk->tkr_mono.mask;
}
diff --git a/arch/arm/vdso/vgettimeofday.c b/arch/arm/vdso/vgettimeofday.c
index a9dd619c6c29..8cf13af1323c 100644
--- a/arch/arm/vdso/vgettimeofday.c
+++ b/arch/arm/vdso/vgettimeofday.c
@@ -1,18 +1,25 @@
/*
- * Copyright 2015 Mentor Graphics Corporation.
+ * Userspace implementations of gettimeofday() and friends.
*
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; version 2 of the
- * License.
+ * Copyright (C) 2017 Cavium, Inc.
+ * Copyright (C) 2015 Mentor Graphics Corporation
+ * Copyright (C) 2012 ARM Limited
*
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * General Public License for more details.
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * Author: Will Deacon <will.dea...@arm.com>
+ * Rewriten from arch64 version into C by: Andrew Pinski <apin...@cavium.com>
+ * Reworked and rebased over arm version by: Mark Salyzyn <saly...@android.com>
*/
#include <linux/compiler.h>
@@ -31,32 +38,30 @@
extern struct vdso_data *__get_datapage(void);
-static notrace u32 __vdso_read_begin(const struct vdso_data *vdata)
-{
- u32 seq;
-repeat:
- seq = READ_ONCE(vdata->seq_count);
- if (seq & 1) {
- cpu_relax();
- goto repeat;
- }
- return seq;
-}
-
static notrace u32 vdso_read_begin(const struct vdso_data *vdata)
{
u32 seq;
- seq = __vdso_read_begin(vdata);
+ do {
+ seq = READ_ONCE(vdata->tb_seq_count);
+
+ if ((seq & 1) == 0)
+ break;
- smp_rmb(); /* Pairs with smp_wmb in vdso_write_end */
+ cpu_relax();
+ } while (true);
+
+ smp_rmb(); /* Pairs with second smp_wmb in update_vsyscall */
return seq;
}
static notrace int vdso_read_retry(const struct vdso_data *vdata, u32 start)
{
- smp_rmb(); /* Pairs with smp_wmb in vdso_write_begin */
- return vdata->seq_count != start;
+ u32 seq;
+
+ smp_rmb(); /* Pairs with first smp_wmb in update_vsyscall */
+ seq = READ_ONCE(vdata->tb_seq_count);
+ return seq != start;
}
static notrace long clock_gettime_fallback(clockid_t _clkid,
@@ -127,7 +132,7 @@ static notrace u64 get_ns(struct vdso_data *vdata)
cycle_delta = (cycle_now - vdata->cs_cycle_last) & vdata->cs_mask;
- nsec = (cycle_delta * vdata->cs_mult) + vdata->xtime_clock_snsec;
+ nsec = (cycle_delta * vdata->cs_mono_mult) + vdata->xtime_clock_snsec;
nsec >>= vdata->cs_shift;
return nsec;
@@ -141,7 +146,7 @@ static notrace int do_realtime(struct timespec *ts, struct
vdso_data *vdata)
do {
seq = vdso_read_begin(vdata);
- if (!vdata->tk_is_cntvct)
+ if (vdata->use_syscall)
return -1;
ts->tv_sec = vdata->xtime_clock_sec;
@@ -164,7 +169,7 @@ static notrace int do_monotonic(struct timespec *ts, struct
vdso_data *vdata)
do {
seq = vdso_read_begin(vdata);
- if (!vdata->tk_is_cntvct)
+ if (vdata->use_syscall)
return -1;
ts->tv_sec = vdata->xtime_clock_sec;
--
2.19.0.605.g01d371f741-goog
