The branch stable/12 has been updated by hselasky:
URL:
https://cgit.FreeBSD.org/src/commit/?id=30b1f8cd1122ef7d770431956c86dd468ae48275
commit 30b1f8cd1122ef7d770431956c86dd468ae48275
Author: Hans Petter Selasky <hsela...@freebsd.org>
AuthorDate: 2021-03-28 07:36:48 +0000
Commit: Hans Petter Selasky <hsela...@freebsd.org>
CommitDate: 2021-04-06 10:25:00 +0000
Reduce chance of RCU deadlock in the LinuxKPI by implementing the section
feature of the concurrency kit, CK.
Differential Revision: https://reviews.freebsd.org/D29467
Reviewed by: kib@ and markj@
Sponsored by: Mellanox Technologies // NVIDIA Networking
(cherry picked from commit 177772088060ab0f41bcdbdd81c4712e7f1c7621)
---
sys/compat/linuxkpi/common/include/linux/sched.h | 1 +
sys/compat/linuxkpi/common/src/linux_rcu.c | 43 ++++++++++++++++++------
2 files changed, 33 insertions(+), 11 deletions(-)
diff --git a/sys/compat/linuxkpi/common/include/linux/sched.h
b/sys/compat/linuxkpi/common/include/linux/sched.h
index da38d89eb639..937e9f27870c 100644
--- a/sys/compat/linuxkpi/common/include/linux/sched.h
+++ b/sys/compat/linuxkpi/common/include/linux/sched.h
@@ -82,6 +82,7 @@ struct task_struct {
int bsd_interrupt_value;
struct work_struct *work; /* current work struct, if set */
struct task_struct *group_leader;
+ unsigned rcu_section[TS_RCU_TYPE_MAX];
};
#define current ({ \
diff --git a/sys/compat/linuxkpi/common/src/linux_rcu.c
b/sys/compat/linuxkpi/common/src/linux_rcu.c
index d2a414926e54..04166926775a 100644
--- a/sys/compat/linuxkpi/common/src/linux_rcu.c
+++ b/sys/compat/linuxkpi/common/src/linux_rcu.c
@@ -1,6 +1,6 @@
/*-
* Copyright (c) 2016 Matthew Macy (mm...@mattmacy.io)
- * Copyright (c) 2017-2020 Hans Petter Selasky (hsela...@freebsd.org)
+ * Copyright (c) 2017-2021 Hans Petter Selasky (hsela...@freebsd.org)
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@@ -85,6 +85,15 @@ struct linux_epoch_record {
*/
CTASSERT(sizeof(struct rcu_head) == sizeof(struct callback_head));
+/*
+ * Verify that "rcu_section[0]" has the same size as
+ * "ck_epoch_section_t". This has been done to avoid having to add
+ * special compile flags for including ck_epoch.h to all clients of
+ * the LinuxKPI.
+ */
+CTASSERT(sizeof(((struct task_struct *)0)->rcu_section[0] ==
+ sizeof(ck_epoch_section_t)));
+
/*
* Verify that "epoch_record" is at beginning of "struct
* linux_epoch_record":
@@ -190,6 +199,14 @@ linux_rcu_read_lock(unsigned type)
if (RCU_SKIP())
return;
+ ts = current;
+
+ /* assert valid refcount */
+ MPASS(ts->rcu_recurse[type] != INT_MAX);
+
+ if (++(ts->rcu_recurse[type]) != 1)
+ return;
+
/*
* Pin thread to current CPU so that the unlock code gets the
* same per-CPU epoch record:
@@ -197,17 +214,15 @@ linux_rcu_read_lock(unsigned type)
sched_pin();
record = &DPCPU_GET(linux_epoch_record[type]);
- ts = current;
/*
* Use a critical section to prevent recursion inside
* ck_epoch_begin(). Else this function supports recursion.
*/
critical_enter();
- ck_epoch_begin(&record->epoch_record, NULL);
- ts->rcu_recurse[type]++;
- if (ts->rcu_recurse[type] == 1)
- TAILQ_INSERT_TAIL(&record->ts_head, ts, rcu_entry[type]);
+ ck_epoch_begin(&record->epoch_record,
+ (ck_epoch_section_t *)&ts->rcu_section[type]);
+ TAILQ_INSERT_TAIL(&record->ts_head, ts, rcu_entry[type]);
critical_exit();
}
@@ -222,18 +237,24 @@ linux_rcu_read_unlock(unsigned type)
if (RCU_SKIP())
return;
- record = &DPCPU_GET(linux_epoch_record[type]);
ts = current;
+ /* assert valid refcount */
+ MPASS(ts->rcu_recurse[type] > 0);
+
+ if (--(ts->rcu_recurse[type]) != 0)
+ return;
+
+ record = &DPCPU_GET(linux_epoch_record[type]);
+
/*
* Use a critical section to prevent recursion inside
* ck_epoch_end(). Else this function supports recursion.
*/
critical_enter();
- ck_epoch_end(&record->epoch_record, NULL);
- ts->rcu_recurse[type]--;
- if (ts->rcu_recurse[type] == 0)
- TAILQ_REMOVE(&record->ts_head, ts, rcu_entry[type]);
+ ck_epoch_end(&record->epoch_record,
+ (ck_epoch_section_t *)&ts->rcu_section[type]);
+ TAILQ_REMOVE(&record->ts_head, ts, rcu_entry[type]);
critical_exit();
sched_unpin();
_______________________________________________
dev-commits-src-all@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/dev-commits-src-all
To unsubscribe, send any mail to "dev-commits-src-all-unsubscr...@freebsd.org"
