Hi,
I would like to use btrace to debug refernce counting. The idea
is to a a tracepoint for every type of refcnt we have. When it
changes, print the actual object, the current counter and the change
value.
#!/usr/sbin/btrace
tracepoint:refcnt:inpcb{
printf("%s %x %u %+d\n", probe, arg0, arg1, arg2)
}
It should look like this:
tracepoint:refcnt:inpcb fffffd8078e31840 0 +1
tracepoint:refcnt:inpcb fffffd8078e31840 1 +1
tracepoint:refcnt:inpcb fffffd8078e31840 2 +1
tracepoint:refcnt:inpcb fffffd8078e31840 3 -1
tracepoint:refcnt:inpcb fffffd8078e31840 2 +1
tracepoint:refcnt:inpcb fffffd8078e31840 3 -1
tracepoint:refcnt:inpcb fffffd8078e31840 2 -1
tracepoint:refcnt:inpcb fffffd8078e31840 1 -1
Unfortunately btrace cannot deal with negative numbers right now.
So it looks like this, but that can be fixed independently.
tracepoint:refcnt:inpcb fffffd8078e31840 0 +1
tracepoint:refcnt:inpcb fffffd8078e31840 1 +1
tracepoint:refcnt:inpcb fffffd8078e31840 2 +1
tracepoint:refcnt:inpcb fffffd8078e31840 3 +4294967295
tracepoint:refcnt:inpcb fffffd8078e31840 2 +1
tracepoint:refcnt:inpcb fffffd8078e31840 3 +4294967295
tracepoint:refcnt:inpcb fffffd8078e31840 2 +4294967295
tracepoint:refcnt:inpcb fffffd8078e31840 1 +4294967295
To debug leaks, btrace can also print kernel stack trace.
tracepoint:refcnt:inpcb fffffd8078e31840 0 +1
in_pcballoc+0x92
tcp_attach+0xd1
sonewconn+0x23d
syn_cache_get+0x1bf
tcp_input+0x885
ip_deliver+0xd3
ip6_input_if+0x762
ipv6_input+0x39
ether_input+0x3a2
if_input_process+0x6f
ifiq_process+0x69
taskq_thread+0xdc
proc_trampoline+0x17
kernel
tracepoint:refcnt:inpcb fffffd8078e31840 1 +1
in_pcbref+0x29
pf_inp_link+0x4e
tcp_input+0x8d2
ip_deliver+0xd3
ip6_input_if+0x762
ipv6_input+0x39
ether_input+0x3a2
if_input_process+0x6f
ifiq_process+0x69
taskq_thread+0xdc
proc_trampoline+0x17
kernel
tracepoint:refcnt:inpcb fffffd8078e31840 2 +1
in_pcbref+0x29
pf_mbuf_link_inpcb+0x27
tcp_output+0x1455
tcp_usrreq+0x386
sosend+0x37c
dofilewritev+0x14d
sys_write+0x51
syscall+0x314
Xsyscall+0x128
kernel
I register the tracepoint when initializing the refcnt. There
exists a global array of possible refcnt types. I implemented it
only for inpcb and tdb as a prove of concept.
Do we want that feature?
bluhm
Index: dev/dt/dt_prov_static.c
===================================================================
RCS file: /data/mirror/openbsd/cvs/src/sys/dev/dt/dt_prov_static.c,v
retrieving revision 1.12
diff -u -p -r1.12 dt_prov_static.c
--- dev/dt/dt_prov_static.c 26 Jan 2022 06:31:31 -0000 1.12
+++ dev/dt/dt_prov_static.c 16 Mar 2022 23:22:34 -0000
@@ -2,6 +2,7 @@
/*
* Copyright (c) 2019 Martin Pieuchot <m...@openbsd.org>
+ * Copyright (c) 2022 Alexander Bluhm <bl...@openbsd.org>
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
@@ -87,6 +88,12 @@ DT_STATIC_PROBE1(smr, barrier_exit, "int
DT_STATIC_PROBE0(smr, wakeup);
DT_STATIC_PROBE2(smr, thread, "uint64_t", "uint64_t");
+/*
+ * reference counting
+ */
+DT_STATIC_PROBE0(refcnt, none);
+DT_STATIC_PROBE3(refcnt, inpcb, "void *", "int", "int");
+DT_STATIC_PROBE3(refcnt, tdb, "void *", "int", "int");
/*
* List of all static probes
@@ -127,15 +134,24 @@ struct dt_probe *dtps_static[] = {
&_DT_STATIC_P(smr, barrier_exit),
&_DT_STATIC_P(smr, wakeup),
&_DT_STATIC_P(smr, thread),
+ /* refcnt */
+ &_DT_STATIC_P(refcnt, none),
+ &_DT_STATIC_P(refcnt, inpcb),
+ &_DT_STATIC_P(refcnt, tdb),
};
+struct dt_probe **dtps_index_refcnt;
+
int
dt_prov_static_init(void)
{
int i;
- for (i = 0; i < nitems(dtps_static); i++)
+ for (i = 0; i < nitems(dtps_static); i++) {
+ if (dtps_static[i] == &_DT_STATIC_P(refcnt, none))
+ dtps_index_refcnt = &dtps_static[i];
dt_dev_register_probe(dtps_static[i]);
+ }
return i;
}
Index: dev/dt/dtvar.h
===================================================================
RCS file: /data/mirror/openbsd/cvs/src/sys/dev/dt/dtvar.h,v
retrieving revision 1.13
diff -u -p -r1.13 dtvar.h
--- dev/dt/dtvar.h 27 Feb 2022 10:14:01 -0000 1.13
+++ dev/dt/dtvar.h 16 Mar 2022 23:22:34 -0000
@@ -2,6 +2,7 @@
/*
* Copyright (c) 2019 Martin Pieuchot <m...@openbsd.org>
+ * Copyright (c) 2022 Alexander Bluhm <bl...@openbsd.org>
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
@@ -313,11 +314,29 @@ extern volatile uint32_t dt_tracing; /*
#define DT_STATIC_ENTER(func, name, args...) do {
\
extern struct dt_probe _DT_STATIC_P(func, name); \
struct dt_probe *dtp = &_DT_STATIC_P(func, name); \
- struct dt_provider *dtpv = dtp->dtp_prov; \
\
if (__predict_false(dt_tracing) && \
__predict_false(dtp->dtp_recording)) { \
+ struct dt_provider *dtpv = dtp->dtp_prov; \
+ \
dtpv->dtpv_enter(dtpv, dtp, args); \
+ } \
+} while (0)
+
+#define _DT_INDEX_P(func) (dtps_index_##func)
+
+#define DT_INDEX_ENTER(func, index, args...) do { \
+ extern struct dt_probe **_DT_INDEX_P(func); \
+ \
+ if (__predict_false(dt_tracing) && \
+ _DT_INDEX_P(func) != NULL && index > 0) { \
+ struct dt_probe *dtp = _DT_INDEX_P(func)[index]; \
+ \
+ if(__predict_false(dtp->dtp_recording)) { \
+ struct dt_provider *dtpv = dtp->dtp_prov; \
+ \
+ dtpv->dtpv_enter(dtpv, dtp, args); \
+ } \
} \
} while (0)
Index: kern/kern_synch.c
===================================================================
RCS file: /data/mirror/openbsd/cvs/src/sys/kern/kern_synch.c,v
retrieving revision 1.184
diff -u -p -r1.184 kern_synch.c
--- kern/kern_synch.c 16 Mar 2022 14:13:01 -0000 1.184
+++ kern/kern_synch.c 16 Mar 2022 23:25:13 -0000
@@ -804,31 +804,37 @@ sys___thrwakeup(struct proc *p, void *v,
void
refcnt_init(struct refcnt *r)
{
+ refcnt_init_trace(r, 0);
+}
+
+void
+refcnt_init_trace(struct refcnt *r, int id)
+{
+ r->r_id = id;
atomic_store_int(&r->r_refs, 1);
+ TRACEINDEX(refcnt, r->r_id, r, 0, +1);
}
void
refcnt_take(struct refcnt *r)
{
-#ifdef DIAGNOSTIC
- u_int refcnt;
+ u_int refs;
- refcnt = atomic_inc_int_nv(&r->r_refs);
- KASSERT(refcnt != 0);
-#else
- atomic_inc_int(&r->r_refs);
-#endif
+ refs = atomic_inc_int_nv(&r->r_refs);
+ KASSERT(refs != 0);
+ TRACEINDEX(refcnt, r->r_id, r, refs - 1, +1);
}
int
refcnt_rele(struct refcnt *r)
{
- u_int refcnt;
+ u_int refs;
- refcnt = atomic_dec_int_nv(&r->r_refs);
- KASSERT(refcnt != ~0);
+ refs = atomic_dec_int_nv(&r->r_refs);
+ KASSERT(refs != ~0);
+ TRACEINDEX(refcnt, r->r_id, r, refs + 1, -1);
- return (refcnt == 0);
+ return (refs == 0);
}
void
@@ -842,26 +848,37 @@ void
refcnt_finalize(struct refcnt *r, const char *wmesg)
{
struct sleep_state sls;
- u_int refcnt;
+ u_int refs;
- refcnt = atomic_dec_int_nv(&r->r_refs);
- while (refcnt) {
+ refs = atomic_dec_int_nv(&r->r_refs);
+ KASSERT(refs != ~0);
+ TRACEINDEX(refcnt, r->r_id, r, refs + 1, -1);
+ while (refs) {
sleep_setup(&sls, r, PWAIT, wmesg, 0);
- refcnt = atomic_load_int(&r->r_refs);
- sleep_finish(&sls, refcnt);
+ refs = atomic_load_int(&r->r_refs);
+ sleep_finish(&sls, refs);
}
+ TRACEINDEX(refcnt, r->r_id, r, refs, 0);
}
int
refcnt_shared(struct refcnt *r)
{
- return (atomic_load_int(&r->r_refs) > 1);
+ u_int refs;
+
+ refs = atomic_load_int(&r->r_refs);
+ TRACEINDEX(refcnt, r->r_id, r, refs, 0);
+ return (refs > 1);
}
unsigned int
refcnt_read(struct refcnt *r)
{
- return (atomic_load_int(&r->r_refs));
+ u_int refs;
+
+ refs = atomic_load_int(&r->r_refs);
+ TRACEINDEX(refcnt, r->r_id, r, refs, 0);
+ return (refs);
}
void
Index: netinet/in_pcb.c
===================================================================
RCS file: /data/mirror/openbsd/cvs/src/sys/netinet/in_pcb.c,v
retrieving revision 1.261
diff -u -p -r1.261 in_pcb.c
--- netinet/in_pcb.c 14 Mar 2022 22:38:43 -0000 1.261
+++ netinet/in_pcb.c 16 Mar 2022 23:22:34 -0000
@@ -234,7 +234,7 @@ in_pcballoc(struct socket *so, struct in
return (ENOBUFS);
inp->inp_table = table;
inp->inp_socket = so;
- refcnt_init(&inp->inp_refcnt);
+ refcnt_init_trace(&inp->inp_refcnt, DT_REFCNT_ID_INPCB);
inp->inp_seclevel[SL_AUTH] = IPSEC_AUTH_LEVEL_DEFAULT;
inp->inp_seclevel[SL_ESP_TRANS] = IPSEC_ESP_TRANS_LEVEL_DEFAULT;
inp->inp_seclevel[SL_ESP_NETWORK] = IPSEC_ESP_NETWORK_LEVEL_DEFAULT;
Index: netinet/ip_ipsp.c
===================================================================
RCS file: /data/mirror/openbsd/cvs/src/sys/netinet/ip_ipsp.c,v
retrieving revision 1.269
diff -u -p -r1.269 ip_ipsp.c
--- netinet/ip_ipsp.c 10 Mar 2022 15:21:08 -0000 1.269
+++ netinet/ip_ipsp.c 16 Mar 2022 23:22:34 -0000
@@ -1048,7 +1048,7 @@ tdb_alloc(u_int rdomain)
tdbp = pool_get(&tdb_pool, PR_WAITOK | PR_ZERO);
- refcnt_init(&tdbp->tdb_refcnt);
+ refcnt_init_trace(&tdbp->tdb_refcnt, DT_REFCNT_ID_TDB);
mtx_init(&tdbp->tdb_mtx, IPL_SOFTNET);
TAILQ_INIT(&tdbp->tdb_policy_head);
Index: sys/refcnt.h
===================================================================
RCS file: /data/mirror/openbsd/cvs/src/sys/sys/refcnt.h,v
retrieving revision 1.6
diff -u -p -r1.6 refcnt.h
--- sys/refcnt.h 16 Mar 2022 14:13:01 -0000 1.6
+++ sys/refcnt.h 16 Mar 2022 23:22:34 -0000
@@ -21,24 +21,30 @@
/*
* Locks used to protect struct members in this file:
+ * I immutable after creation
* a atomic operations
*/
struct refcnt {
+ int r_id; /* [I] index for dt(4) tracing */
unsigned int r_refs; /* [a] reference counter */
};
-#define REFCNT_INITIALIZER() { .r_refs = 1 }
+#define REFCNT_INITIALIZER() { .r_id = 0, .r_refs = 1 }
#ifdef _KERNEL
void refcnt_init(struct refcnt *);
+void refcnt_init_trace(struct refcnt *, int id);
void refcnt_take(struct refcnt *);
int refcnt_rele(struct refcnt *);
void refcnt_rele_wake(struct refcnt *);
void refcnt_finalize(struct refcnt *, const char *);
int refcnt_shared(struct refcnt *);
unsigned int refcnt_read(struct refcnt *);
+
+#define DT_REFCNT_ID_INPCB 1
+#define DT_REFCNT_ID_TDB 2
#endif /* _KERNEL */
Index: sys/tracepoint.h
===================================================================
RCS file: /data/mirror/openbsd/cvs/src/sys/sys/tracepoint.h,v
retrieving revision 1.1
diff -u -p -r1.1 tracepoint.h
--- sys/tracepoint.h 21 Jan 2020 16:16:23 -0000 1.1
+++ sys/tracepoint.h 16 Mar 2022 23:22:34 -0000
@@ -25,11 +25,13 @@
#if NDT > 0
#include <dev/dt/dtvar.h>
-#define TRACEPOINT(func, name, args...) DT_STATIC_ENTER(func, name,
args)
+#define TRACEPOINT(func, name, args...) DT_STATIC_ENTER(func, name,
args)
+#define TRACEINDEX(func, index, args...) DT_INDEX_ENTER(func, index, args)
#else /* NDT > 0 */
-#define TRACEPOINT(func, name, args...)
+#define TRACEPOINT(func, name, args...)
+#define TRACEINDEX(func, index, args...)
#endif /* NDT > 0 */
#endif /* _KERNEL */
