Author: cem
Date: Tue May 3 17:07:18 2016
New Revision: 298989
URL: https://svnweb.freebsd.org/changeset/base/298989
Log:
ioat(4): Implement CRC and MOVECRC APIs
And document them in ioat.4.
Sponsored by: EMC / Isilon Storage Division
Modified:
head/share/man/man4/ioat.4
head/sys/dev/ioat/ioat.c
head/sys/dev/ioat/ioat.h
Modified: head/share/man/man4/ioat.4
==============================================================================
--- head/share/man/man4/ioat.4 Tue May 3 17:06:33 2016 (r298988)
+++ head/share/man/man4/ioat.4 Tue May 3 17:07:18 2016 (r298989)
@@ -24,7 +24,7 @@
.\"
.\" $FreeBSD$
.\"
-.Dd January 14, 2016
+.Dd May 3, 2016
.Dt IOAT 4
.Os
.Sh NAME
@@ -99,6 +99,29 @@ In
.Fa "uint32_t flags"
.Fc
.Ft struct bus_dmadesc *
+.Fo ioat_copy_crc
+.Fa "bus_dmaengine_t dmaengine"
+.Fa "bus_addr_t dst"
+.Fa "bus_addr_t src"
+.Fa "bus_size_t len"
+.Fa "uint32_t *initialseed"
+.Fa "bus_addr_t crcptr"
+.Fa "bus_dmaengine_callback_t callback_fn"
+.Fa "void *callback_arg"
+.Fa "uint32_t flags"
+.Fc
+.Ft struct bus_dmadesc *
+.Fo ioat_crc
+.Fa "bus_dmaengine_t dmaengine"
+.Fa "bus_addr_t src"
+.Fa "bus_size_t len"
+.Fa "uint32_t *initialseed"
+.Fa "bus_addr_t crcptr"
+.Fa "bus_dmaengine_callback_t callback_fn"
+.Fa "void *callback_arg"
+.Fa "uint32_t flags"
+.Fc
+.Ft struct bus_dmadesc *
.Fo ioat_blockfill
.Fa "bus_dmaengine_t dmaengine"
.Fa "bus_addr_t dst"
@@ -183,6 +206,43 @@ from.
It is invalid to attempt to submit new DMA operations in a
.Fa bus_dmaengine_callback_t
context.
+.Pp
+The CRC operations have three distinct modes.
+The default mode is to accumulate.
+By accumulating over multiple descriptors, a user may gather a CRC over several
+chunks of memory and only write out the result once.
+.Pp
+The
+.Ar DMA_CRC_STORE
+flag causes the operation to emit the CRC32C result.
+If
+.Ar DMA_CRC_INLINE
+is set, the result is written inline with the destination data (or source in
+.Fn ioat_crc
+mode).
+If
+.Ar DMA_CRC_INLINE
+is not set, the result is written to the provided
+.Fa crcptr .
+.Pp
+Similarly, the
+.Ar DMA_CRC_TEST
+flag causes the operation to compare the CRC32C result to an existing checksum.
+If
+.Ar DMA_CRC_INLINE
+is set, the result is compared against the inline four bytes trailing the
+source data.
+If it is not set, the result is compared against the value pointed to by
+.Fa crcptr .
+.Pp
+.Fn ioat_copy_crc
+calculates a CRC32C while copying data.
+.Fn ioat_crc
+only computes a CRC32C of some data.
+If the
+.Fa initialseed
+argument to either routine is non-NULL, the CRC32C engine is initialized with
+the value it points to.
.Sh USAGE
A typical user will lookup the DMA engine object for a given channel with
.Fn ioat_get_dmaengine .
@@ -199,9 +259,13 @@ If
succeeds, there is guaranteed to be room for
.Fa N
new operations in the internal ring buffer.
+.Pp
Then, they will submit one or more operations using
.Fn ioat_blockfill ,
.Fn ioat_copy ,
+.Fn ioat_copy_8k_aligned ,
+.Fn ioat_copy_crc ,
+.Fn ioat_crc ,
or
.Fn ioat_null .
After queuing one or more individual DMA operations, they will
Modified: head/sys/dev/ioat/ioat.c
==============================================================================
--- head/sys/dev/ioat/ioat.c Tue May 3 17:06:33 2016 (r298988)
+++ head/sys/dev/ioat/ioat.c Tue May 3 17:07:18 2016 (r298989)
@@ -882,8 +882,8 @@ ioat_op_generic(struct ioat_softc *ioat,
mtx_assert(&ioat->submit_lock, MA_OWNED);
- KASSERT((flags & ~DMA_ALL_FLAGS) == 0, ("Unrecognized flag(s): %#x",
- flags & ~DMA_ALL_FLAGS));
+ KASSERT((flags & ~_DMA_GENERIC_FLAGS) == 0,
+ ("Unrecognized flag(s): %#x", flags & ~_DMA_GENERIC_FLAGS));
if ((flags & DMA_NO_WAIT) != 0)
mflags = M_NOWAIT;
else
@@ -1018,6 +1018,164 @@ ioat_copy_8k_aligned(bus_dmaengine_t dma
}
struct bus_dmadesc *
+ioat_copy_crc(bus_dmaengine_t dmaengine, bus_addr_t dst, bus_addr_t src,
+ bus_size_t len, uint32_t *initialseed, bus_addr_t crcptr,
+ bus_dmaengine_callback_t callback_fn, void *callback_arg, uint32_t flags)
+{
+ struct ioat_crc32_hw_descriptor *hw_desc;
+ struct ioat_descriptor *desc;
+ struct ioat_softc *ioat;
+ uint32_t teststore;
+ uint8_t op;
+
+ CTR0(KTR_IOAT, __func__);
+ ioat = to_ioat_softc(dmaengine);
+
+ if ((ioat->capabilities & IOAT_DMACAP_MOVECRC) == 0) {
+ ioat_log_message(0, "%s: Device lacks MOVECRC capability\n",
+ __func__);
+ return (NULL);
+ }
+ if (((src | dst) & (0xffffffull << 40)) != 0) {
+ ioat_log_message(0, "%s: High 24 bits of src/dst invalid\n",
+ __func__);
+ return (NULL);
+ }
+ teststore = (flags & _DMA_CRC_TESTSTORE);
+ if (teststore == _DMA_CRC_TESTSTORE) {
+ ioat_log_message(0, "%s: TEST and STORE invalid\n", __func__);
+ return (NULL);
+ }
+ if (teststore == 0 && (flags & DMA_CRC_INLINE) != 0) {
+ ioat_log_message(0, "%s: INLINE invalid without TEST or
STORE\n",
+ __func__);
+ return (NULL);
+ }
+
+ switch (teststore) {
+ case DMA_CRC_STORE:
+ op = IOAT_OP_MOVECRC_STORE;
+ break;
+ case DMA_CRC_TEST:
+ op = IOAT_OP_MOVECRC_TEST;
+ break;
+ default:
+ KASSERT(teststore == 0, ("bogus"));
+ op = IOAT_OP_MOVECRC;
+ break;
+ }
+
+ if ((flags & DMA_CRC_INLINE) == 0 &&
+ (crcptr & (0xffffffull << 40)) != 0) {
+ ioat_log_message(0,
+ "%s: High 24 bits of crcptr invalid\n", __func__);
+ return (NULL);
+ }
+
+ desc = ioat_op_generic(ioat, op, len, src, dst, callback_fn,
+ callback_arg, flags & ~_DMA_CRC_FLAGS);
+ if (desc == NULL)
+ return (NULL);
+
+ hw_desc = desc->u.crc32;
+
+ if ((flags & DMA_CRC_INLINE) == 0)
+ hw_desc->crc_address = crcptr;
+ else
+ hw_desc->u.control.crc_location = 1;
+
+ if (initialseed != NULL) {
+ hw_desc->u.control.use_seed = 1;
+ hw_desc->seed = *initialseed;
+ }
+
+ if (g_ioat_debug_level >= 3)
+ dump_descriptor(hw_desc);
+
+ ioat_submit_single(ioat);
+ return (&desc->bus_dmadesc);
+}
+
+struct bus_dmadesc *
+ioat_crc(bus_dmaengine_t dmaengine, bus_addr_t src, bus_size_t len,
+ uint32_t *initialseed, bus_addr_t crcptr,
+ bus_dmaengine_callback_t callback_fn, void *callback_arg, uint32_t flags)
+{
+ struct ioat_crc32_hw_descriptor *hw_desc;
+ struct ioat_descriptor *desc;
+ struct ioat_softc *ioat;
+ uint32_t teststore;
+ uint8_t op;
+
+ CTR0(KTR_IOAT, __func__);
+ ioat = to_ioat_softc(dmaengine);
+
+ if ((ioat->capabilities & IOAT_DMACAP_CRC) == 0) {
+ ioat_log_message(0, "%s: Device lacks CRC capability\n",
+ __func__);
+ return (NULL);
+ }
+ if ((src & (0xffffffull << 40)) != 0) {
+ ioat_log_message(0, "%s: High 24 bits of src invalid\n",
+ __func__);
+ return (NULL);
+ }
+ teststore = (flags & _DMA_CRC_TESTSTORE);
+ if (teststore == _DMA_CRC_TESTSTORE) {
+ ioat_log_message(0, "%s: TEST and STORE invalid\n", __func__);
+ return (NULL);
+ }
+ if (teststore == 0 && (flags & DMA_CRC_INLINE) != 0) {
+ ioat_log_message(0, "%s: INLINE invalid without TEST or
STORE\n",
+ __func__);
+ return (NULL);
+ }
+
+ switch (teststore) {
+ case DMA_CRC_STORE:
+ op = IOAT_OP_CRC_STORE;
+ break;
+ case DMA_CRC_TEST:
+ op = IOAT_OP_CRC_TEST;
+ break;
+ default:
+ KASSERT(teststore == 0, ("bogus"));
+ op = IOAT_OP_CRC;
+ break;
+ }
+
+ if ((flags & DMA_CRC_INLINE) == 0 &&
+ (crcptr & (0xffffffull << 40)) != 0) {
+ ioat_log_message(0,
+ "%s: High 24 bits of crcptr invalid\n", __func__);
+ return (NULL);
+ }
+
+ desc = ioat_op_generic(ioat, op, len, src, 0, callback_fn,
+ callback_arg, flags & ~_DMA_CRC_FLAGS);
+ if (desc == NULL)
+ return (NULL);
+
+ hw_desc = desc->u.crc32;
+
+ if ((flags & DMA_CRC_INLINE) == 0)
+ hw_desc->crc_address = crcptr;
+ else
+ hw_desc->u.control.crc_location = 1;
+
+ if (initialseed != NULL) {
+ hw_desc->u.control.use_seed = 1;
+ hw_desc->seed = *initialseed;
+ }
+
+ if (g_ioat_debug_level >= 3)
+ dump_descriptor(hw_desc);
+
+ ioat_submit_single(ioat);
+ return (&desc->bus_dmadesc);
+}
+
+struct bus_dmadesc *
ioat_blockfill(bus_dmaengine_t dmaengine, bus_addr_t dst, uint64_t fillpattern,
bus_size_t len, bus_dmaengine_callback_t callback_fn, void *callback_arg,
uint32_t flags)
Modified: head/sys/dev/ioat/ioat.h
==============================================================================
--- head/sys/dev/ioat/ioat.h Tue May 3 17:06:33 2016 (r298988)
+++ head/sys/dev/ioat/ioat.h Tue May 3 17:07:18 2016 (r298989)
@@ -52,7 +52,26 @@ __FBSDID("$FreeBSD$");
* may be prefetched before operation 1 completes.
*/
#define DMA_FENCE 0x4
-#define DMA_ALL_FLAGS (DMA_INT_EN | DMA_NO_WAIT | DMA_FENCE)
+#define _DMA_GENERIC_FLAGS (DMA_INT_EN | DMA_NO_WAIT | DMA_FENCE)
+
+/*
+ * Emit a CRC32C as the result of a ioat_copy_crc() or ioat_crc().
+ */
+#define DMA_CRC_STORE 0x8
+
+/*
+ * Compare the CRC32C of a ioat_copy_crc() or ioat_crc() against an expeceted
+ * value. It is invalid to specify both TEST and STORE.
+ */
+#define DMA_CRC_TEST 0x10
+#define _DMA_CRC_TESTSTORE (DMA_CRC_STORE | DMA_CRC_TEST)
+
+/*
+ * Use an inline comparison CRC32C or emit an inline CRC32C result. Invalid
+ * without one of STORE or TEST.
+ */
+#define DMA_CRC_INLINE 0x20
+#define _DMA_CRC_FLAGS (DMA_CRC_STORE | DMA_CRC_TEST | DMA_CRC_INLINE)
/*
* Hardware revision number. Different hardware revisions support different
@@ -152,6 +171,42 @@ struct bus_dmadesc *ioat_copy_8k_aligned
bus_dmaengine_callback_t callback_fn, void *callback_arg, uint32_t flags);
/*
+ * Copy len bytes from dst to src, like ioat_copy().
+ *
+ * Additionally, accumulate a CRC32C of the data.
+ *
+ * If initialseed is not NULL, the value it points to is used to seed the
+ * initial value of the CRC32C.
+ *
+ * If flags include DMA_CRC_STORE and not DMA_CRC_INLINE, crcptr is written
+ * with the 32-bit CRC32C result (in wire format).
+ *
+ * If flags include DMA_CRC_TEST and not DMA_CRC_INLINE, the computed CRC32C is
+ * compared with the 32-bit CRC32C pointed to by crcptr. If they do not match,
+ * a channel error is raised.
+ *
+ * If the DMA_CRC_INLINE flag is set, crcptr is ignored and the DMA engine uses
+ * the 4 bytes trailing the source data (TEST) or the destination data (STORE).
+ */
+struct bus_dmadesc *ioat_copy_crc(bus_dmaengine_t dmaengine, bus_addr_t dst,
+ bus_addr_t src, bus_size_t len, uint32_t *initialseed, bus_addr_t crcptr,
+ bus_dmaengine_callback_t callback_fn, void *callback_arg, uint32_t flags);
+
+/*
+ * ioat_crc() is nearly identical to ioat_copy_crc(), but does not actually
+ * move data around.
+ *
+ * Like ioat_copy_crc, ioat_crc computes a CRC32C over len bytes pointed to by
+ * src. The flags affect its operation in the same way, with one exception:
+ *
+ * If flags includes both DMA_CRC_STORE and DMA_CRC_INLINE, the computed CRC32C
+ * is written to the 4 bytes trailing the *source* data.
+ */
+struct bus_dmadesc *ioat_crc(bus_dmaengine_t dmaengine, bus_addr_t src,
+ bus_size_t len, uint32_t *initialseed, bus_addr_t crcptr,
+ bus_dmaengine_callback_t callback_fn, void *callback_arg, uint32_t flags);
+
+/*
* Issues a null operation. This issues the operation to the hardware, but the
* hardware doesn't do anything with it.
*/
_______________________________________________
svn-src-all@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-all
To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"
