Module Name: src
Committed By: riastradh
Date: Sun Dec 19 00:27:01 UTC 2021
Added Files:
src/sys/external/bsd/drm2/include/linux: dma-fence.h
src/sys/external/bsd/drm2/linux: linux_dma_fence.c
Removed Files:
src/sys/external/bsd/drm2/include/linux: fence.h
src/sys/external/bsd/drm2/linux: linux_fence.c
Log Message:
Rename fence -> dma_fence, step 1: files.
To generate a diff of this commit:
cvs rdiff -u -r0 -r1.1 src/sys/external/bsd/drm2/include/linux/dma-fence.h
cvs rdiff -u -r1.16 -r0 src/sys/external/bsd/drm2/include/linux/fence.h
cvs rdiff -u -r0 -r1.1 src/sys/external/bsd/drm2/linux/linux_dma_fence.c
cvs rdiff -u -r1.16 -r0 src/sys/external/bsd/drm2/linux/linux_fence.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Added files:
Index: src/sys/external/bsd/drm2/include/linux/dma-fence.h
diff -u /dev/null src/sys/external/bsd/drm2/include/linux/dma-fence.h:1.1
--- /dev/null Sun Dec 19 00:27:01 2021
+++ src/sys/external/bsd/drm2/include/linux/dma-fence.h Sun Dec 19 00:27:01 2021
@@ -0,0 +1,143 @@
+/* $NetBSD: dma-fence.h,v 1.1 2021/12/19 00:27:01 riastradh Exp $ */
+
+/*-
+ * Copyright (c) 2018 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Taylor R. Campbell.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _LINUX_FENCE_H_
+#define _LINUX_FENCE_H_
+
+#include <sys/types.h>
+#include <sys/condvar.h>
+#include <sys/kernel.h>
+#include <sys/queue.h>
+
+#include <linux/kref.h>
+#include <linux/rcupdate.h>
+#include <linux/spinlock.h>
+
+struct fence_cb;
+
+struct fence {
+ struct kref refcount;
+ spinlock_t *lock;
+ volatile unsigned long flags;
+ unsigned context;
+ unsigned seqno;
+ const struct fence_ops *ops;
+
+ TAILQ_HEAD(, fence_cb) f_callbacks;
+ kcondvar_t f_cv;
+ struct rcu_head f_rcu;
+};
+
+#define FENCE_FLAG_ENABLE_SIGNAL_BIT 0
+#define FENCE_FLAG_SIGNALED_BIT 1
+#define FENCE_FLAG_USER_BITS 2
+
+struct fence_ops {
+ const char *(*get_driver_name)(struct fence *);
+ const char *(*get_timeline_name)(struct fence *);
+ bool (*enable_signaling)(struct fence *);
+ bool (*signaled)(struct fence *);
+ long (*wait)(struct fence *, bool, long);
+ void (*release)(struct fence *);
+};
+
+typedef void (*fence_func_t)(struct fence *, struct fence_cb *);
+
+struct fence_cb {
+ fence_func_t fcb_func;
+ TAILQ_ENTRY(fence_cb) fcb_entry;
+ bool fcb_onqueue;
+};
+
+#define fence_add_callback linux_fence_add_callback
+#define fence_context_alloc linux_fence_context_alloc
+#define fence_default_wait linux_fence_default_wait
+#define fence_destroy linux_fence_destroy
+#define fence_enable_sw_signaling linux_fence_enable_sw_signaling
+#define fence_free linux_fence_free
+#define fence_get linux_fence_get
+#define fence_get_rcu linux_fence_get_rcu
+#define fence_init linux_fence_init
+#define fence_is_later linux_fence_is_later
+#define fence_is_signaled linux_fence_is_signaled
+#define fence_is_signaled_locked linux_fence_is_signaled_locked
+#define fence_put linux_fence_put
+#define fence_remove_callback linux_fence_remove_callback
+#define fence_signal linux_fence_signal
+#define fence_signal_locked linux_fence_signal_locked
+#define fence_wait linux_fence_wait
+#define fence_wait_any_timeout linux_fence_wait_any_timeout
+#define fence_wait_timeout linux_fence_wait_timeout
+
+extern int linux_fence_trace;
+
+void fence_init(struct fence *, const struct fence_ops *, spinlock_t *,
+ unsigned, unsigned);
+void fence_destroy(struct fence *);
+void fence_free(struct fence *);
+
+unsigned
+ fence_context_alloc(unsigned);
+bool fence_is_later(struct fence *, struct fence *);
+
+struct fence *
+ fence_get(struct fence *);
+struct fence *
+ fence_get_rcu(struct fence *);
+void fence_put(struct fence *);
+
+int fence_add_callback(struct fence *, struct fence_cb *, fence_func_t);
+bool fence_remove_callback(struct fence *, struct fence_cb *);
+void fence_enable_sw_signaling(struct fence *);
+
+bool fence_is_signaled(struct fence *);
+bool fence_is_signaled_locked(struct fence *);
+int fence_signal(struct fence *);
+int fence_signal_locked(struct fence *);
+long fence_default_wait(struct fence *, bool, long);
+long fence_wait(struct fence *, bool);
+long fence_wait_any_timeout(struct fence **, uint32_t, bool, long);
+long fence_wait_timeout(struct fence *, bool, long);
+
+static inline void __printflike(2, 3)
+FENCE_TRACE(struct fence *f, const char *fmt, ...)
+{
+ va_list va;
+
+ if (__predict_false(linux_fence_trace)) {
+ va_start(va, fmt);
+ printf("fence %u@%u: ", f->context, f->seqno);
+ vprintf(fmt, va);
+ va_end(va);
+ }
+}
+
+#endif /* _LINUX_FENCE_H_ */
Index: src/sys/external/bsd/drm2/linux/linux_dma_fence.c
diff -u /dev/null src/sys/external/bsd/drm2/linux/linux_dma_fence.c:1.1
--- /dev/null Sun Dec 19 00:27:01 2021
+++ src/sys/external/bsd/drm2/linux/linux_dma_fence.c Sun Dec 19 00:27:01 2021
@@ -0,0 +1,752 @@
+/* $NetBSD: linux_dma_fence.c,v 1.1 2021/12/19 00:27:01 riastradh Exp $ */
+
+/*-
+ * Copyright (c) 2018 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Taylor R. Campbell.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__KERNEL_RCSID(0, "$NetBSD: linux_dma_fence.c,v 1.1 2021/12/19 00:27:01 riastradh Exp $");
+
+#include <sys/atomic.h>
+#include <sys/condvar.h>
+#include <sys/queue.h>
+
+#include <linux/atomic.h>
+#include <linux/errno.h>
+#include <linux/kref.h>
+#include <linux/fence.h>
+#include <linux/sched.h>
+#include <linux/spinlock.h>
+
+/*
+ * linux_fence_trace
+ *
+ * True if we print FENCE_TRACE messages, false if not. These are
+ * extremely noisy, too much even for AB_VERBOSE and AB_DEBUG in
+ * boothowto.
+ */
+int linux_fence_trace = 0;
+
+/*
+ * fence_referenced_p(fence)
+ *
+ * True if fence has a positive reference count. True after
+ * fence_init; after the last fence_put, this becomes false.
+ */
+static inline bool __diagused
+fence_referenced_p(struct fence *fence)
+{
+
+ return kref_referenced_p(&fence->refcount);
+}
+
+/*
+ * fence_init(fence, ops, lock, context, seqno)
+ *
+ * Initialize fence. Caller should call fence_destroy when done,
+ * after all references have been released.
+ */
+void
+fence_init(struct fence *fence, const struct fence_ops *ops, spinlock_t *lock,
+ unsigned context, unsigned seqno)
+{
+
+ kref_init(&fence->refcount);
+ fence->lock = lock;
+ fence->flags = 0;
+ fence->context = context;
+ fence->seqno = seqno;
+ fence->ops = ops;
+ TAILQ_INIT(&fence->f_callbacks);
+ cv_init(&fence->f_cv, "fence");
+}
+
+/*
+ * fence_destroy(fence)
+ *
+ * Clean up memory initialized with fence_init. This is meant to
+ * be used after a fence release callback.
+ */
+void
+fence_destroy(struct fence *fence)
+{
+
+ KASSERT(!fence_referenced_p(fence));
+
+ KASSERT(TAILQ_EMPTY(&fence->f_callbacks));
+ cv_destroy(&fence->f_cv);
+}
+
+static void
+fence_free_cb(struct rcu_head *rcu)
+{
+ struct fence *fence = container_of(rcu, struct fence, f_rcu);
+
+ KASSERT(!fence_referenced_p(fence));
+
+ fence_destroy(fence);
+ kfree(fence);
+}
+
+/*
+ * fence_free(fence)
+ *
+ * Schedule fence to be destroyed and then freed with kfree after
+ * any pending RCU read sections on all CPUs have completed.
+ * Caller must guarantee all references have been released. This
+ * is meant to be used after a fence release callback.
+ *
+ * NOTE: Callers assume kfree will be used. We don't even use
+ * kmalloc to allocate these -- caller is expected to allocate
+ * memory with kmalloc to be initialized with fence_init.
+ */
+void
+fence_free(struct fence *fence)
+{
+
+ KASSERT(!fence_referenced_p(fence));
+
+ call_rcu(&fence->f_rcu, &fence_free_cb);
+}
+
+/*
+ * fence_context_alloc(n)
+ *
+ * Return the first of a contiguous sequence of unique
+ * identifiers, at least until the system wraps around.
+ */
+unsigned
+fence_context_alloc(unsigned n)
+{
+ static volatile unsigned next_context = 0;
+
+ return atomic_add_int_nv(&next_context, n) - n;
+}
+
+/*
+ * fence_is_later(a, b)
+ *
+ * True if the sequence number of fence a is later than the
+ * sequence number of fence b. Since sequence numbers wrap
+ * around, we define this to mean that the sequence number of
+ * fence a is no more than INT_MAX past the sequence number of
+ * fence b.
+ *
+ * The two fences must have the same context.
+ */
+bool
+fence_is_later(struct fence *a, struct fence *b)
+{
+
+ KASSERTMSG(a->context == b->context, "incommensurate fences"
+ ": %u @ %p =/= %u @ %p", a->context, a, b->context, b);
+
+ return a->seqno - b->seqno < INT_MAX;
+}
+
+/*
+ * fence_get(fence)
+ *
+ * Acquire a reference to fence. The fence must not be being
+ * destroyed. Return the fence.
+ */
+struct fence *
+fence_get(struct fence *fence)
+{
+
+ if (fence)
+ kref_get(&fence->refcount);
+ return fence;
+}
+
+/*
+ * fence_get_rcu(fence)
+ *
+ * Attempt to acquire a reference to a fence that may be about to
+ * be destroyed, during a read section. Return the fence on
+ * success, or NULL on failure.
+ */
+struct fence *
+fence_get_rcu(struct fence *fence)
+{
+
+ if (!kref_get_unless_zero(&fence->refcount))
+ return NULL;
+ return fence;
+}
+
+static void
+fence_release(struct kref *refcount)
+{
+ struct fence *fence = container_of(refcount, struct fence, refcount);
+
+ KASSERT(!fence_referenced_p(fence));
+
+ if (fence->ops->release)
+ (*fence->ops->release)(fence);
+ else
+ fence_free(fence);
+}
+
+/*
+ * fence_put(fence)
+ *
+ * Release a reference to fence. If this was the last one, call
+ * the fence's release callback.
+ */
+void
+fence_put(struct fence *fence)
+{
+
+ if (fence == NULL)
+ return;
+ KASSERT(fence_referenced_p(fence));
+ kref_put(&fence->refcount, &fence_release);
+}
+
+/*
+ * fence_ensure_signal_enabled(fence)
+ *
+ * Internal subroutine. If the fence was already signalled,
+ * return -ENOENT. Otherwise, if the enable signalling callback
+ * has not been called yet, call it. If fails, signal the fence
+ * and return -ENOENT. If it succeeds, or if it had already been
+ * called, return zero to indicate success.
+ *
+ * Caller must hold the fence's lock.
+ */
+static int
+fence_ensure_signal_enabled(struct fence *fence)
+{
+
+ KASSERT(fence_referenced_p(fence));
+ KASSERT(spin_is_locked(fence->lock));
+
+ /* If the fence was already signalled, fail with -ENOENT. */
+ if (fence->flags & (1u << FENCE_FLAG_SIGNALED_BIT))
+ return -ENOENT;
+
+ /*
+ * If the enable signaling callback has been called, success.
+ * Otherwise, set the bit indicating it.
+ */
+ if (test_and_set_bit(FENCE_FLAG_ENABLE_SIGNAL_BIT, &fence->flags))
+ return 0;
+
+ /* Otherwise, note that we've called it and call it. */
+ if (!(*fence->ops->enable_signaling)(fence)) {
+ /* If it failed, signal and return -ENOENT. */
+ fence_signal_locked(fence);
+ return -ENOENT;
+ }
+
+ /* Success! */
+ return 0;
+}
+
+/*
+ * fence_add_callback(fence, fcb, fn)
+ *
+ * If fence has been signalled, return -ENOENT. If the enable
+ * signalling callback hasn't been called yet, call it; if it
+ * fails, return -ENOENT. Otherwise, arrange to call fn(fence,
+ * fcb) when it is signalled, and return 0.
+ *
+ * The fence uses memory allocated by the caller in fcb from the
+ * time of fence_add_callback either to the time of
+ * fence_remove_callback, or just before calling fn.
+ */
+int
+fence_add_callback(struct fence *fence, struct fence_cb *fcb, fence_func_t fn)
+{
+ int ret;
+
+ KASSERT(fence_referenced_p(fence));
+
+ /* Optimistically try to skip the lock if it's already signalled. */
+ if (fence->flags & (1u << FENCE_FLAG_SIGNALED_BIT)) {
+ ret = -ENOENT;
+ goto out0;
+ }
+
+ /* Acquire the lock. */
+ spin_lock(fence->lock);
+
+ /* Ensure signalling is enabled, or fail if we can't. */
+ ret = fence_ensure_signal_enabled(fence);
+ if (ret)
+ goto out1;
+
+ /* Insert the callback. */
+ fcb->fcb_func = fn;
+ TAILQ_INSERT_TAIL(&fence->f_callbacks, fcb, fcb_entry);
+ fcb->fcb_onqueue = true;
+
+ /* Release the lock and we're done. */
+out1: spin_unlock(fence->lock);
+out0: return ret;
+}
+
+/*
+ * fence_remove_callback(fence, fcb)
+ *
+ * Remove the callback fcb from fence. Return true if it was
+ * removed from the list, or false if it had already run and so
+ * was no longer queued anyway. Caller must have already called
+ * fence_add_callback(fence, fcb).
+ */
+bool
+fence_remove_callback(struct fence *fence, struct fence_cb *fcb)
+{
+ bool onqueue;
+
+ KASSERT(fence_referenced_p(fence));
+
+ spin_lock(fence->lock);
+ onqueue = fcb->fcb_onqueue;
+ if (onqueue) {
+ TAILQ_REMOVE(&fence->f_callbacks, fcb, fcb_entry);
+ fcb->fcb_onqueue = false;
+ }
+ spin_unlock(fence->lock);
+
+ return onqueue;
+}
+
+/*
+ * fence_enable_sw_signaling(fence)
+ *
+ * If it hasn't been called yet and the fence hasn't been
+ * signalled yet, call the fence's enable_sw_signaling callback.
+ * If when that happens, the callback indicates failure by
+ * returning false, signal the fence.
+ */
+void
+fence_enable_sw_signaling(struct fence *fence)
+{
+
+ KASSERT(fence_referenced_p(fence));
+
+ spin_lock(fence->lock);
+ (void)fence_ensure_signal_enabled(fence);
+ spin_unlock(fence->lock);
+}
+
+/*
+ * fence_is_signaled(fence)
+ *
+ * Test whether the fence has been signalled. If it has been
+ * signalled by fence_signal(_locked), return true. If the
+ * signalled callback returns true indicating that some implicit
+ * external condition has changed, call the callbacks as if with
+ * fence_signal.
+ */
+bool
+fence_is_signaled(struct fence *fence)
+{
+ bool signaled;
+
+ KASSERT(fence_referenced_p(fence));
+
+ spin_lock(fence->lock);
+ signaled = fence_is_signaled_locked(fence);
+ spin_unlock(fence->lock);
+
+ return signaled;
+}
+
+/*
+ * fence_is_signaled_locked(fence)
+ *
+ * Test whether the fence has been signalled. Like
+ * fence_is_signaleed, but caller already holds the fence's lock.
+ */
+bool
+fence_is_signaled_locked(struct fence *fence)
+{
+
+ KASSERT(fence_referenced_p(fence));
+ KASSERT(spin_is_locked(fence->lock));
+
+ /* Check whether we already set the signalled bit. */
+ if (fence->flags & (1u << FENCE_FLAG_SIGNALED_BIT))
+ return true;
+
+ /* If there's a signalled callback, test it. */
+ if (fence->ops->signaled) {
+ if ((*fence->ops->signaled)(fence)) {
+ /*
+ * It's been signalled implicitly by some
+ * external phenomonen. Act as though someone
+ * has called fence_signal.
+ */
+ fence_signal_locked(fence);
+ return true;
+ }
+ }
+
+ return false;
+}
+
+/*
+ * fence_signal(fence)
+ *
+ * Signal the fence. If it has already been signalled, return
+ * -EINVAL. If it has not been signalled, call the enable
+ * signalling callback if it hasn't been called yet, and remove
+ * each registered callback from the queue and call it; then
+ * return 0.
+ */
+int
+fence_signal(struct fence *fence)
+{
+ int ret;
+
+ KASSERT(fence_referenced_p(fence));
+
+ spin_lock(fence->lock);
+ ret = fence_signal_locked(fence);
+ spin_unlock(fence->lock);
+
+ return ret;
+}
+
+/*
+ * fence_signal_locked(fence)
+ *
+ * Signal the fence. Like fence_signal, but caller already holds
+ * the fence's lock.
+ */
+int
+fence_signal_locked(struct fence *fence)
+{
+ struct fence_cb *fcb, *next;
+
+ KASSERT(fence_referenced_p(fence));
+ KASSERT(spin_is_locked(fence->lock));
+
+ /* If it's been signalled, fail; otherwise set the signalled bit. */
+ if (test_and_set_bit(FENCE_FLAG_SIGNALED_BIT, &fence->flags))
+ return -EINVAL;
+
+ /* Wake waiters. */
+ cv_broadcast(&fence->f_cv);
+
+ /* Remove and call the callbacks. */
+ TAILQ_FOREACH_SAFE(fcb, &fence->f_callbacks, fcb_entry, next) {
+ TAILQ_REMOVE(&fence->f_callbacks, fcb, fcb_entry);
+ fcb->fcb_onqueue = false;
+ (*fcb->fcb_func)(fence, fcb);
+ }
+
+ /* Success! */
+ return 0;
+}
+
+struct wait_any {
+ struct fence_cb fcb;
+ struct wait_any1 {
+ kmutex_t lock;
+ kcondvar_t cv;
+ bool done;
+ } *common;
+};
+
+static void
+wait_any_cb(struct fence *fence, struct fence_cb *fcb)
+{
+ struct wait_any *cb = container_of(fcb, struct wait_any, fcb);
+
+ KASSERT(fence_referenced_p(fence));
+
+ mutex_enter(&cb->common->lock);
+ cb->common->done = true;
+ cv_broadcast(&cb->common->cv);
+ mutex_exit(&cb->common->lock);
+}
+
+/*
+ * fence_wait_any_timeout(fence, nfences, intr, timeout)
+ *
+ * Wait for any of fences[0], fences[1], fences[2], ...,
+ * fences[nfences-1] to be signaled.
+ */
+long
+fence_wait_any_timeout(struct fence **fences, uint32_t nfences, bool intr,
+ long timeout)
+{
+ struct wait_any1 common;
+ struct wait_any *cb;
+ uint32_t i, j;
+ int start, end;
+ long ret = 0;
+
+ /* Allocate an array of callback records. */
+ cb = kcalloc(nfences, sizeof(cb[0]), GFP_KERNEL);
+ if (cb == NULL) {
+ ret = -ENOMEM;
+ goto out0;
+ }
+
+ /* Initialize a mutex and condvar for the common wait. */
+ mutex_init(&common.lock, MUTEX_DEFAULT, IPL_VM);
+ cv_init(&common.cv, "fence");
+ common.done = false;
+
+ /* Add a callback to each of the fences, or stop here if we can't. */
+ for (i = 0; i < nfences; i++) {
+ cb[i].common = &common;
+ KASSERT(fence_referenced_p(fences[i]));
+ ret = fence_add_callback(fences[i], &cb[i].fcb, &wait_any_cb);
+ if (ret)
+ goto out1;
+ }
+
+ /*
+ * Test whether any of the fences has been signalled. If they
+ * have, stop here. If the haven't, we are guaranteed to be
+ * notified by one of the callbacks when they have.
+ */
+ for (j = 0; j < nfences; j++) {
+ if (test_bit(FENCE_FLAG_SIGNALED_BIT, &fences[j]->flags))
+ goto out1;
+ }
+
+ /*
+ * None of them was ready immediately. Wait for one of the
+ * callbacks to notify us when it is done.
+ */
+ mutex_enter(&common.lock);
+ while (timeout > 0 && !common.done) {
+ start = getticks();
+ __insn_barrier();
+ if (intr) {
+ if (timeout != MAX_SCHEDULE_TIMEOUT) {
+ ret = -cv_timedwait_sig(&common.cv,
+ &common.lock, MIN(timeout, /* paranoia */
+ MAX_SCHEDULE_TIMEOUT));
+ } else {
+ ret = -cv_wait_sig(&common.cv, &common.lock);
+ }
+ } else {
+ if (timeout != MAX_SCHEDULE_TIMEOUT) {
+ ret = -cv_timedwait(&common.cv,
+ &common.lock, MIN(timeout, /* paranoia */
+ MAX_SCHEDULE_TIMEOUT));
+ } else {
+ cv_wait(&common.cv, &common.lock);
+ ret = 0;
+ }
+ }
+ end = getticks();
+ __insn_barrier();
+ if (ret) {
+ if (ret == -ERESTART)
+ ret = -ERESTARTSYS;
+ break;
+ }
+ timeout -= MIN(timeout, (unsigned)end - (unsigned)start);
+ }
+ mutex_exit(&common.lock);
+
+ /*
+ * Massage the return code: if we were interrupted, return
+ * ERESTARTSYS; if cv_timedwait timed out, return 0; otherwise
+ * return the remaining time.
+ */
+ if (ret < 0) {
+ if (ret == -EINTR || ret == -ERESTART)
+ ret = -ERESTARTSYS;
+ if (ret == -EWOULDBLOCK)
+ ret = 0;
+ } else {
+ KASSERT(ret == 0);
+ ret = timeout;
+ }
+
+out1: while (i --> 0)
+ (void)fence_remove_callback(fences[i], &cb[i].fcb);
+ cv_destroy(&common.cv);
+ mutex_destroy(&common.lock);
+ kfree(cb);
+out0: return ret;
+}
+
+/*
+ * fence_wait_timeout(fence, intr, timeout)
+ *
+ * Wait until fence is signalled; or until interrupt, if intr is
+ * true; or until timeout, if positive. Return -ERESTARTSYS if
+ * interrupted, negative error code on any other error, zero on
+ * timeout, or positive number of ticks remaining if the fence is
+ * signalled before the timeout. Works by calling the fence wait
+ * callback.
+ *
+ * The timeout must be nonnegative and less than
+ * MAX_SCHEDULE_TIMEOUT.
+ */
+long
+fence_wait_timeout(struct fence *fence, bool intr, long timeout)
+{
+
+ KASSERT(fence_referenced_p(fence));
+ KASSERT(timeout >= 0);
+ KASSERT(timeout < MAX_SCHEDULE_TIMEOUT);
+
+ return (*fence->ops->wait)(fence, intr, timeout);
+}
+
+/*
+ * fence_wait(fence, intr)
+ *
+ * Wait until fence is signalled; or until interrupt, if intr is
+ * true. Return -ERESTARTSYS if interrupted, negative error code
+ * on any other error, zero on sucess. Works by calling the fence
+ * wait callback with MAX_SCHEDULE_TIMEOUT.
+ */
+long
+fence_wait(struct fence *fence, bool intr)
+{
+ long ret;
+
+ KASSERT(fence_referenced_p(fence));
+
+ ret = (*fence->ops->wait)(fence, intr, MAX_SCHEDULE_TIMEOUT);
+ KASSERT(ret != 0);
+
+ return (ret < 0 ? ret : 0);
+}
+
+/*
+ * fence_default_wait(fence, intr, timeout)
+ *
+ * Default implementation of fence wait callback using a condition
+ * variable. If the fence is already signalled, return timeout,
+ * or 1 if no timeout. If the enable signalling callback hasn't
+ * been called, call it, and if it fails, act as if the fence had
+ * been signalled. Otherwise, wait on the internal condvar. If
+ * timeout is MAX_SCHEDULE_TIMEOUT, treat it as no timeout.
+ */
+long
+fence_default_wait(struct fence *fence, bool intr, long timeout)
+{
+ int starttime = 0, now = 0, deadline = 0; /* XXXGCC */
+ kmutex_t *lock = &fence->lock->sl_lock;
+ long ret = 0;
+
+ KASSERT(fence_referenced_p(fence));
+ KASSERTMSG(timeout >= 0, "timeout %ld", timeout);
+ KASSERTMSG(timeout <= MAX_SCHEDULE_TIMEOUT, "timeout %ld", timeout);
+
+ /* Optimistically try to skip the lock if it's already signalled. */
+ if (fence->flags & (1u << FENCE_FLAG_SIGNALED_BIT))
+ return (timeout < MAX_SCHEDULE_TIMEOUT ? timeout : 1);
+
+ /* Acquire the lock. */
+ spin_lock(fence->lock);
+
+ /* Ensure signalling is enabled, or fail if we can't. */
+ ret = fence_ensure_signal_enabled(fence);
+ if (ret)
+ goto out;
+
+ /* Find out what our deadline is so we can handle spurious wakeup. */
+ if (timeout < MAX_SCHEDULE_TIMEOUT) {
+ now = getticks();
+ __insn_barrier();
+ starttime = now;
+ deadline = starttime + timeout;
+ }
+
+ /* Wait until the signalled bit is set. */
+ while (!(fence->flags & (1u << FENCE_FLAG_SIGNALED_BIT))) {
+ /*
+ * If there's a timeout and we've passed the deadline,
+ * give up.
+ */
+ if (timeout < MAX_SCHEDULE_TIMEOUT) {
+ now = getticks();
+ __insn_barrier();
+ if (deadline <= now)
+ break;
+ }
+ if (intr) {
+ if (timeout < MAX_SCHEDULE_TIMEOUT) {
+ ret = -cv_timedwait_sig(&fence->f_cv, lock,
+ deadline - now);
+ } else {
+ ret = -cv_wait_sig(&fence->f_cv, lock);
+ }
+ } else {
+ if (timeout < MAX_SCHEDULE_TIMEOUT) {
+ ret = -cv_timedwait(&fence->f_cv, lock,
+ deadline - now);
+ } else {
+ cv_wait(&fence->f_cv, lock);
+ ret = 0;
+ }
+ }
+ /* If the wait failed, give up. */
+ if (ret) {
+ if (ret == -ERESTART)
+ ret = -ERESTARTSYS;
+ break;
+ }
+ }
+
+out:
+ /* All done. Release the lock. */
+ spin_unlock(fence->lock);
+
+ /* If cv_timedwait gave up, return 0 meaning timeout. */
+ if (ret == -EWOULDBLOCK) {
+ /* Only cv_timedwait and cv_timedwait_sig can return this. */
+ KASSERT(timeout < MAX_SCHEDULE_TIMEOUT);
+ return 0;
+ }
+
+ /* If there was a timeout and the deadline passed, return 0. */
+ if (timeout < MAX_SCHEDULE_TIMEOUT) {
+ if (deadline <= now)
+ return 0;
+ }
+
+ /* If we were interrupted, return -ERESTARTSYS. */
+ if (ret == -EINTR || ret == -ERESTART)
+ return -ERESTARTSYS;
+
+ /* If there was any other kind of error, fail. */
+ if (ret)
+ return ret;
+
+ /*
+ * Success! Return the number of ticks left, at least 1, or 1
+ * if no timeout.
+ */
+ return (timeout < MAX_SCHEDULE_TIMEOUT ? MIN(deadline - now, 1) : 1);
+}
