The branch main has been updated by markj:
URL:
https://cgit.FreeBSD.org/src/commit/?id=bd674d8b1fa808dff1afdcb6afc70226077ee78a
commit bd674d8b1fa808dff1afdcb6afc70226077ee78a
Author: Mark Johnston <ma...@freebsd.org>
AuthorDate: 2021-01-27 20:30:58 +0000
Commit: Mark Johnston <ma...@freebsd.org>
CommitDate: 2021-01-27 20:30:58 +0000
qat: Add support for separate AAD and output buffers
MFC after: 1 week
Sponsored by: Rubicon Communications, LLC ("Netgate")
---
sys/dev/qat/qat.c | 293 +++++++++++++++++++++++++++++++++++++------------
sys/dev/qat/qat_hw15.c | 20 +++-
sys/dev/qat/qat_hw17.c | 22 +++-
sys/dev/qat/qatreg.h | 3 +
sys/dev/qat/qatvar.h | 43 ++++----
5 files changed, 281 insertions(+), 100 deletions(-)
diff --git a/sys/dev/qat/qat.c b/sys/dev/qat/qat.c
index 1e533d1bebcc..acfe0c6a1e07 100644
--- a/sys/dev/qat/qat.c
+++ b/sys/dev/qat/qat.c
@@ -1172,9 +1172,8 @@ static void
qat_crypto_free_sym_cookie(struct qat_crypto_bank *qcb,
struct qat_sym_cookie *qsc)
{
-
- explicit_bzero(qsc->qsc_iv_buf, sizeof(qsc->qsc_iv_buf));
- explicit_bzero(qsc->qsc_auth_res, sizeof(qsc->qsc_auth_res));
+ explicit_bzero(qsc->qsc_iv_buf, EALG_MAX_BLOCK_LEN);
+ explicit_bzero(qsc->qsc_auth_res, QAT_SYM_HASH_BUFFER_LEN);
mtx_lock(&qcb->qcb_bank_mtx);
qcb->qcb_symck_free[qcb->qcb_symck_free_count++] = qsc;
@@ -1350,18 +1349,66 @@ struct qat_crypto_load_cb_arg {
int error;
};
+static int
+qat_crypto_populate_buf_list(struct buffer_list_desc *buffers,
+ bus_dma_segment_t *segs, int niseg, int noseg, int skip)
+{
+ struct flat_buffer_desc *flatbuf;
+ bus_addr_t addr;
+ bus_size_t len;
+ int iseg, oseg;
+
+ for (iseg = 0, oseg = noseg; iseg < niseg && oseg < QAT_MAXSEG;
+ iseg++) {
+ addr = segs[iseg].ds_addr;
+ len = segs[iseg].ds_len;
+
+ if (skip > 0) {
+ if (skip < len) {
+ addr += skip;
+ len -= skip;
+ skip = 0;
+ } else {
+ skip -= len;
+ continue;
+ }
+ }
+
+ flatbuf = &buffers->flat_bufs[oseg++];
+ flatbuf->data_len_in_bytes = (uint32_t)len;
+ flatbuf->phy_buffer = (uint64_t)addr;
+ }
+ buffers->num_buffers = oseg;
+ return iseg < niseg ? E2BIG : 0;
+}
+
static void
-qat_crypto_load_cb(void *_arg, bus_dma_segment_t *segs, int nseg,
+qat_crypto_load_aadbuf_cb(void *_arg, bus_dma_segment_t *segs, int nseg,
+ int error)
+{
+ struct qat_crypto_load_cb_arg *arg;
+ struct qat_sym_cookie *qsc;
+
+ arg = _arg;
+ if (error != 0) {
+ arg->error = error;
+ return;
+ }
+
+ qsc = arg->qsc;
+ arg->error = qat_crypto_populate_buf_list(&qsc->qsc_buf_list, segs,
+ nseg, 0, 0);
+}
+
+static void
+qat_crypto_load_buf_cb(void *_arg, bus_dma_segment_t *segs, int nseg,
int error)
{
struct cryptop *crp;
- struct flat_buffer_desc *flatbuf;
struct qat_crypto_load_cb_arg *arg;
struct qat_session *qs;
struct qat_sym_cookie *qsc;
- bus_addr_t addr;
- bus_size_t len;
- int iseg, oseg, skip;
+ int noseg, skip;
arg = _arg;
if (error != 0) {
@@ -1374,43 +1421,70 @@ qat_crypto_load_cb(void *_arg, bus_dma_segment_t *segs,
int nseg,
qsc = arg->qsc;
if (qs->qs_auth_algo == HW_AUTH_ALGO_GALOIS_128) {
- /*
- * The firmware expects AAD to be in a contiguous buffer and
- * padded to a multiple of 16 bytes. To satisfy these
- * constraints we bounce the AAD into a per-request buffer.
- */
- crypto_copydata(crp, crp->crp_aad_start, crp->crp_aad_length,
- qsc->qsc_gcm_aad);
- memset(qsc->qsc_gcm_aad + crp->crp_aad_length, 0,
- roundup2(crp->crp_aad_length, QAT_AES_GCM_AAD_ALIGN) -
- crp->crp_aad_length);
+ /* AAD was handled in qat_crypto_load(). */
skip = crp->crp_payload_start;
- } else if (crp->crp_aad_length > 0) {
+ noseg = 0;
+ } else if (crp->crp_aad == NULL && crp->crp_aad_length > 0) {
skip = crp->crp_aad_start;
+ noseg = 0;
} else {
skip = crp->crp_payload_start;
+ noseg = crp->crp_aad == NULL ?
+ 0 : qsc->qsc_buf_list.num_buffers;
}
+ arg->error = qat_crypto_populate_buf_list(&qsc->qsc_buf_list, segs,
+ nseg, noseg, skip);
+}
- for (iseg = oseg = 0; iseg < nseg; iseg++) {
- addr = segs[iseg].ds_addr;
- len = segs[iseg].ds_len;
+static void
+qat_crypto_load_obuf_cb(void *_arg, bus_dma_segment_t *segs, int nseg,
+ int error)
+{
+ struct buffer_list_desc *ibufs, *obufs;
+ struct flat_buffer_desc *ibuf, *obuf;
+ struct cryptop *crp;
+ struct qat_crypto_load_cb_arg *arg;
+ struct qat_session *qs;
+ struct qat_sym_cookie *qsc;
+ int buflen, osegs, tocopy;
- if (skip > 0) {
- if (skip < len) {
- addr += skip;
- len -= skip;
- skip = 0;
- } else {
- skip -= len;
- continue;
- }
- }
+ arg = _arg;
+ if (error != 0) {
+ arg->error = error;
+ return;
+ }
- flatbuf = &qsc->qsc_flat_bufs[oseg++];
- flatbuf->data_len_in_bytes = (uint32_t)len;
- flatbuf->phy_buffer = (uint64_t)addr;
+ crp = arg->crp;
+ qs = arg->qs;
+ qsc = arg->qsc;
+
+ /*
+ * The payload must start at the same offset in the output SG list as in
+ * the input SG list. Copy over SG entries from the input corresponding
+ * to the AAD buffer.
+ */
+ osegs = 0;
+ if (qs->qs_auth_algo != HW_AUTH_ALGO_GALOIS_128 &&
+ crp->crp_aad_length > 0) {
+ tocopy = crp->crp_aad == NULL ?
+ crp->crp_payload_start - crp->crp_aad_start :
+ crp->crp_aad_length;
+
+ ibufs = &qsc->qsc_buf_list;
+ obufs = &qsc->qsc_obuf_list;
+ for (; osegs < ibufs->num_buffers && tocopy > 0; osegs++) {
+ ibuf = &ibufs->flat_bufs[osegs];
+ obuf = &obufs->flat_bufs[osegs];
+
+ obuf->phy_buffer = ibuf->phy_buffer;
+ buflen = imin(ibuf->data_len_in_bytes, tocopy);
+ obuf->data_len_in_bytes = buflen;
+ tocopy -= buflen;
+ }
}
- qsc->qsc_buf_list.num_buffers = oseg;
+
+ arg->error = qat_crypto_populate_buf_list(&qsc->qsc_obuf_list, segs,
+ nseg, osegs, crp->crp_payload_output_start);
}
static int
@@ -1426,10 +1500,52 @@ qat_crypto_load(struct qat_session *qs, struct
qat_sym_cookie *qsc,
arg.qs = qs;
arg.qsc = qsc;
arg.error = 0;
- error = bus_dmamap_load_crp(qsc->qsc_buf_dma_tag, qsc->qsc_buf_dmamap,
- crp, qat_crypto_load_cb, &arg, BUS_DMA_NOWAIT);
- if (error == 0)
- error = arg.error;
+
+ error = 0;
+ if (qs->qs_auth_algo == HW_AUTH_ALGO_GALOIS_128 &&
+ crp->crp_aad_length > 0) {
+ /*
+ * The firmware expects AAD to be in a contiguous buffer and
+ * padded to a multiple of 16 bytes. To satisfy these
+ * constraints we bounce the AAD into a per-request buffer.
+ * There is a small limit on the AAD size so this is not too
+ * onerous.
+ */
+ memset(qsc->qsc_gcm_aad, 0, QAT_GCM_AAD_SIZE_MAX);
+ if (crp->crp_aad == NULL) {
+ crypto_copydata(crp, crp->crp_aad_start,
+ crp->crp_aad_length, qsc->qsc_gcm_aad);
+ } else {
+ memcpy(qsc->qsc_gcm_aad, crp->crp_aad,
+ crp->crp_aad_length);
+ }
+ } else if (crp->crp_aad != NULL) {
+ error = bus_dmamap_load(
+ qsc->qsc_dma[QAT_SYM_DMA_AADBUF].qsd_dma_tag,
+ qsc->qsc_dma[QAT_SYM_DMA_AADBUF].qsd_dmamap,
+ crp->crp_aad, crp->crp_aad_length,
+ qat_crypto_load_aadbuf_cb, &arg, BUS_DMA_NOWAIT);
+ if (error == 0)
+ error = arg.error;
+ }
+ if (error == 0) {
+ error = bus_dmamap_load_crp_buffer(
+ qsc->qsc_dma[QAT_SYM_DMA_BUF].qsd_dma_tag,
+ qsc->qsc_dma[QAT_SYM_DMA_BUF].qsd_dmamap,
+ &crp->crp_buf, qat_crypto_load_buf_cb, &arg,
+ BUS_DMA_NOWAIT);
+ if (error == 0)
+ error = arg.error;
+ }
+ if (error == 0 && CRYPTO_HAS_OUTPUT_BUFFER(crp)) {
+ error = bus_dmamap_load_crp_buffer(
+ qsc->qsc_dma[QAT_SYM_DMA_OBUF].qsd_dma_tag,
+ qsc->qsc_dma[QAT_SYM_DMA_OBUF].qsd_dmamap,
+ &crp->crp_obuf, qat_crypto_load_obuf_cb, &arg,
+ BUS_DMA_NOWAIT);
+ if (error == 0)
+ error = arg.error;
+ }
return error;
}
@@ -1444,11 +1560,11 @@ qat_crypto_select_bank(struct qat_crypto *qcy)
static int
qat_crypto_setup_ring(struct qat_softc *sc, struct qat_crypto_bank *qcb)
{
- int error, i, bank;
- int curname = 0;
char *name;
+ int bank, curname, error, i, j;
bank = qcb->qcb_bank;
+ curname = 0;
name = qcb->qcb_ring_names[curname++];
snprintf(name, QAT_RING_NAME_SIZE, "bank%d sym_tx", bank);
@@ -1480,10 +1596,16 @@ qat_crypto_setup_ring(struct qat_softc *sc, struct
qat_crypto_bank *qcb)
qsc->qsc_self_dma_tag = qdm->qdm_dma_tag;
qsc->qsc_bulk_req_params_buf_paddr =
qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie,
- u.qsc_bulk_cookie.qsbc_req_params_buf);
+ qsc_bulk_cookie.qsbc_req_params_buf);
qsc->qsc_buffer_list_desc_paddr =
qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie,
qsc_buf_list);
+ qsc->qsc_obuffer_list_desc_paddr =
+ qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie,
+ qsc_obuf_list);
+ qsc->qsc_obuffer_list_desc_paddr =
+ qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie,
+ qsc_obuf_list);
qsc->qsc_iv_buf_paddr =
qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie,
qsc_iv_buf);
@@ -1499,24 +1621,25 @@ qat_crypto_setup_ring(struct qat_softc *sc, struct
qat_crypto_bank *qcb)
qcb->qcb_symck_free[i] = qsc;
qcb->qcb_symck_free_count++;
- error = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev),
- 1, 0, /* alignment, boundary */
- BUS_SPACE_MAXADDR, /* lowaddr */
- BUS_SPACE_MAXADDR, /* highaddr */
- NULL, NULL, /* filter, filterarg */
- QAT_MAXLEN, /* maxsize */
- QAT_MAXSEG, /* nsegments */
- QAT_MAXLEN, /* maxsegsize */
- BUS_DMA_COHERENT, /* flags */
- NULL, NULL, /* lockfunc, lockarg */
- &qsc->qsc_buf_dma_tag);
- if (error != 0)
- return error;
-
- error = bus_dmamap_create(qsc->qsc_buf_dma_tag,
- BUS_DMA_COHERENT, &qsc->qsc_buf_dmamap);
- if (error)
- return error;
+ for (j = 0; j < QAT_SYM_DMA_COUNT; j++) {
+ error = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev),
+ 1, 0, /* alignment, boundary */
+ BUS_SPACE_MAXADDR, /* lowaddr */
+ BUS_SPACE_MAXADDR, /* highaddr */
+ NULL, NULL, /* filter, filterarg */
+ QAT_MAXLEN, /* maxsize */
+ QAT_MAXSEG, /* nsegments */
+ QAT_MAXLEN, /* maxsegsize */
+ BUS_DMA_COHERENT, /* flags */
+ NULL, NULL, /* lockfunc, lockarg */
+ &qsc->qsc_dma[j].qsd_dma_tag);
+ if (error != 0)
+ return error;
+ error = bus_dmamap_create(qsc->qsc_dma[j].qsd_dma_tag,
+ BUS_DMA_COHERENT, &qsc->qsc_dma[j].qsd_dmamap);
+ if (error != 0)
+ return error;
+ }
}
return 0;
@@ -1534,10 +1657,17 @@ static void
qat_crypto_bank_deinit(struct qat_softc *sc, struct qat_crypto_bank *qcb)
{
struct qat_dmamem *qdm;
- int i;
+ struct qat_sym_cookie *qsc;
+ int i, j;
for (i = 0; i < QAT_NSYMCOOKIE; i++) {
qdm = &qcb->qcb_symck_dmamems[i];
+ qsc = qcb->qcb_symck_free[i];
+ for (j = 0; j < QAT_SYM_DMA_COUNT; j++) {
+ bus_dmamap_destroy(qsc->qsc_dma[j].qsd_dma_tag,
+ qsc->qsc_dma[j].qsd_dmamap);
+ bus_dma_tag_destroy(qsc->qsc_dma[j].qsd_dma_tag);
+ }
qat_free_dmamem(sc, qdm);
}
qat_free_dmamem(sc, &qcb->qcb_sym_tx->qr_dma);
@@ -1653,6 +1783,15 @@ qat_crypto_stop(struct qat_softc *sc)
(void)crypto_unregister_all(qcy->qcy_cid);
}
+static void
+qat_crypto_sym_dma_unload(struct qat_sym_cookie *qsc, enum qat_sym_dma i)
+{
+ bus_dmamap_sync(qsc->qsc_dma[i].qsd_dma_tag, qsc->qsc_dma[i].qsd_dmamap,
+ BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
+ bus_dmamap_unload(qsc->qsc_dma[i].qsd_dma_tag,
+ qsc->qsc_dma[i].qsd_dmamap);
+}
+
static int
qat_crypto_sym_rxintr(struct qat_softc *sc, void *arg, void *msg)
{
@@ -1669,16 +1808,19 @@ qat_crypto_sym_rxintr(struct qat_softc *sc, void *arg,
void *msg)
qsc = *(void **)((uintptr_t)msg + sc->sc_hw.qhw_crypto_opaque_offset);
- qsbc = &qsc->u.qsc_bulk_cookie;
+ qsbc = &qsc->qsc_bulk_cookie;
qcy = qsbc->qsbc_crypto;
qs = qsbc->qsbc_session;
crp = qsbc->qsbc_cb_tag;
bus_dmamap_sync(qsc->qsc_self_dma_tag, qsc->qsc_self_dmamap,
BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
- bus_dmamap_sync(qsc->qsc_buf_dma_tag, qsc->qsc_buf_dmamap,
- BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
- bus_dmamap_unload(qsc->qsc_buf_dma_tag, qsc->qsc_buf_dmamap);
+
+ if (crp->crp_aad != NULL)
+ qat_crypto_sym_dma_unload(qsc, QAT_SYM_DMA_AADBUF);
+ qat_crypto_sym_dma_unload(qsc, QAT_SYM_DMA_BUF);
+ if (CRYPTO_HAS_OUTPUT_BUFFER(crp))
+ qat_crypto_sym_dma_unload(qsc, QAT_SYM_DMA_OBUF);
error = 0;
if ((auth_sz = qs->qs_auth_mlen) != 0) {
@@ -1719,6 +1861,10 @@ qat_crypto_sym_rxintr(struct qat_softc *sc, void *arg,
void *msg)
static int
qat_probesession(device_t dev, const struct crypto_session_params *csp)
{
+ if ((csp->csp_flags & ~(CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD)) !=
+ 0)
+ return EINVAL;
+
if (csp->csp_cipher_alg == CRYPTO_AES_XTS &&
qat_lookup(dev)->qatp_chip == QAT_CHIP_C2XXX) {
/*
@@ -2092,15 +2238,26 @@ qat_process(device_t dev, struct cryptop *crp, int hint)
if (error != 0)
goto fail2;
- qsbc = &qsc->u.qsc_bulk_cookie;
+ qsbc = &qsc->qsc_bulk_cookie;
qsbc->qsbc_crypto = qcy;
qsbc->qsbc_session = qs;
qsbc->qsbc_cb_tag = crp;
sc->sc_hw.qhw_crypto_setup_req_params(qcb, qs, desc, qsc, crp);
- bus_dmamap_sync(qsc->qsc_buf_dma_tag, qsc->qsc_buf_dmamap,
+ if (crp->crp_aad != NULL) {
+ bus_dmamap_sync(qsc->qsc_dma[QAT_SYM_DMA_AADBUF].qsd_dma_tag,
+ qsc->qsc_dma[QAT_SYM_DMA_AADBUF].qsd_dmamap,
+ BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
+ }
+ bus_dmamap_sync(qsc->qsc_dma[QAT_SYM_DMA_BUF].qsd_dma_tag,
+ qsc->qsc_dma[QAT_SYM_DMA_BUF].qsd_dmamap,
BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
+ if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) {
+ bus_dmamap_sync(qsc->qsc_dma[QAT_SYM_DMA_OBUF].qsd_dma_tag,
+ qsc->qsc_dma[QAT_SYM_DMA_OBUF].qsd_dmamap,
+ BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
+ }
bus_dmamap_sync(qsc->qsc_self_dma_tag, qsc->qsc_self_dmamap,
BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
diff --git a/sys/dev/qat/qat_hw15.c b/sys/dev/qat/qat_hw15.c
index 4f823a0a9ae4..aab5be9b647d 100644
--- a/sys/dev/qat/qat_hw15.c
+++ b/sys/dev/qat/qat_hw15.c
@@ -818,7 +818,7 @@ qat_hw15_crypto_setup_req_params(struct qat_crypto_bank
*qcb,
enum fw_la_cmd_id cmd_id = desc->qcd_cmd_id;
enum fw_slice next_slice;
- qsbc = &qsc->u.qsc_bulk_cookie;
+ qsbc = &qsc->qsc_bulk_cookie;
bulk_req = (struct fw_la_bulk_req *)qsbc->qsbc_msg;
memcpy(bulk_req, &desc->qcd_req_cache, QAT_HW15_SESSION_REQ_CACHE_SIZE);
@@ -826,7 +826,13 @@ qat_hw15_crypto_setup_req_params(struct qat_crypto_bank
*qcb,
bulk_req->comn_hdr.comn_req_flags =
qat_hw15_get_comn_req_flags(qcb->qcb_bank % 2);
bulk_req->comn_mid.src_data_addr = qsc->qsc_buffer_list_desc_paddr;
- bulk_req->comn_mid.dest_data_addr = qsc->qsc_buffer_list_desc_paddr;
+ if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) {
+ bulk_req->comn_mid.dest_data_addr =
+ qsc->qsc_obuffer_list_desc_paddr;
+ } else {
+ bulk_req->comn_mid.dest_data_addr =
+ qsc->qsc_buffer_list_desc_paddr;
+ }
bulk_req->req_params_addr = qsc->qsc_bulk_req_params_buf_paddr;
bulk_req->comn_ftr.next_request_addr = 0;
bulk_req->comn_mid.opaque_data = (uint64_t)(uintptr_t)qsc;
@@ -918,8 +924,14 @@ qat_hw15_crypto_setup_req_params(struct qat_crypto_bank
*qcb,
cipher_req->curr_id = FW_SLICE_CIPHER;
cipher_req->next_id = next_slice;
- cipher_req->cipher_off = crp->crp_aad_length == 0 ? 0 :
- crp->crp_payload_start - crp->crp_aad_start;
+ if (crp->crp_aad_length == 0) {
+ cipher_req->cipher_off = 0;
+ } else if (crp->crp_aad == NULL) {
+ cipher_req->cipher_off =
+ crp->crp_payload_start - crp->crp_aad_start;
+ } else {
+ cipher_req->cipher_off = crp->crp_aad_length;
+ }
cipher_req->cipher_len = crp->crp_payload_length;
cipher_req->state_address = qsc->qsc_iv_buf_paddr;
}
diff --git a/sys/dev/qat/qat_hw17.c b/sys/dev/qat/qat_hw17.c
index 643b624ba840..92f2c1ee251d 100644
--- a/sys/dev/qat/qat_hw17.c
+++ b/sys/dev/qat/qat_hw17.c
@@ -581,13 +581,19 @@ qat_hw17_crypto_setup_req_params(struct qat_crypto_bank
*qcb __unused,
uint8_t *req_params_ptr;
enum fw_la_cmd_id cmd_id = desc->qcd_cmd_id;
- qsbc = &qsc->u.qsc_bulk_cookie;
+ qsbc = &qsc->qsc_bulk_cookie;
bulk_req = (struct fw_la_bulk_req *)qsbc->qsbc_msg;
memcpy(bulk_req, desc->qcd_req_cache, sizeof(struct fw_la_bulk_req));
bulk_req->comn_mid.opaque_data = (uint64_t)(uintptr_t)qsc;
bulk_req->comn_mid.src_data_addr = qsc->qsc_buffer_list_desc_paddr;
- bulk_req->comn_mid.dest_data_addr = qsc->qsc_buffer_list_desc_paddr;
+ if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) {
+ bulk_req->comn_mid.dest_data_addr =
+ qsc->qsc_obuffer_list_desc_paddr;
+ } else {
+ bulk_req->comn_mid.dest_data_addr =
+ qsc->qsc_buffer_list_desc_paddr;
+ }
if (__predict_false(crp->crp_cipher_key != NULL ||
crp->crp_auth_key != NULL))
qat_hw17_crypto_req_setkey(desc, qs, qsc, bulk_req, crp);
@@ -643,9 +649,15 @@ qat_hw17_crypto_setup_req_params(struct qat_crypto_bank
*qcb __unused,
}
} else {
if (cmd_id != FW_LA_CMD_AUTH) {
- cipher_param->cipher_offset =
- crp->crp_aad_length == 0 ? 0 :
- crp->crp_payload_start - crp->crp_aad_start;
+ if (crp->crp_aad_length == 0) {
+ cipher_param->cipher_offset = 0;
+ } else if (crp->crp_aad == NULL) {
+ cipher_param->cipher_offset =
+ crp->crp_payload_start - crp->crp_aad_start;
+ } else {
+ cipher_param->cipher_offset =
+ crp->crp_aad_length;
+ }
cipher_param->cipher_length = crp->crp_payload_length;
}
if (cmd_id != FW_LA_CMD_CIPHER) {
diff --git a/sys/dev/qat/qatreg.h b/sys/dev/qat/qatreg.h
index 2a0be321b4f9..076354fc6108 100644
--- a/sys/dev/qat/qatreg.h
+++ b/sys/dev/qat/qatreg.h
@@ -1550,10 +1550,13 @@ struct flat_buffer_desc {
uint64_t phy_buffer;
};
+#define HW_MAXSEG 32
+
struct buffer_list_desc {
uint64_t resrvd;
uint32_t num_buffers;
uint32_t reserved;
+ struct flat_buffer_desc flat_bufs[HW_MAXSEG];
};
/* --------------------------------------------------------------------------
*/
diff --git a/sys/dev/qat/qatvar.h b/sys/dev/qat/qatvar.h
index 3326dc268fc1..3f0ecd411874 100644
--- a/sys/dev/qat/qatvar.h
+++ b/sys/dev/qat/qatvar.h
@@ -69,13 +69,11 @@
#define QAT_NSYMREQ 256
#define QAT_NSYMCOOKIE ((QAT_NSYMREQ * 2 + 1) * 2)
-#define QAT_NASYMREQ 64
-#define QAT_BATCH_SUBMIT_FREE_SPACE 2
#define QAT_EV_NAME_SIZE 32
#define QAT_RING_NAME_SIZE 32
-#define QAT_MAXSEG 32 /* max segments for sg dma */
+#define QAT_MAXSEG HW_MAXSEG /* max segments for sg dma */
#define QAT_MAXLEN 65535 /* IP_MAXPACKET */
#define QAT_HB_INTERVAL 500 /* heartbeat msec */
@@ -519,7 +517,7 @@ struct qat_sym_hash_def {
struct qat_sym_bulk_cookie {
uint8_t qsbc_req_params_buf[QAT_SYM_REQ_PARAMS_SIZE_PADDED];
- /* memory block reserved for request params
+ /* memory block reserved for request params, QAT 1.5 only
* NOTE: Field must be correctly aligned in memory for access by QAT
* engine */
struct qat_crypto *qsbc_crypto;
@@ -539,25 +537,26 @@ struct qat_sym_bulk_cookie {
HASH_CONTENT_DESC_SIZE + CIPHER_CONTENT_DESC_SIZE, \
QAT_OPTIMAL_ALIGN)
+enum qat_sym_dma {
+ QAT_SYM_DMA_AADBUF = 0,
+ QAT_SYM_DMA_BUF,
+ QAT_SYM_DMA_OBUF,
+ QAT_SYM_DMA_COUNT,
+};
+
+struct qat_sym_dmamap {
+ bus_dmamap_t qsd_dmamap;
+ bus_dma_tag_t qsd_dma_tag;
+};
+
struct qat_sym_cookie {
- union qat_sym_cookie_u {
- /* should be 64byte aligned */
- struct qat_sym_bulk_cookie qsc_bulk_cookie;
- /* symmetric bulk cookie */
-#ifdef notyet
- struct qat_sym_key_cookie qsc_key_cookie;
- /* symmetric key cookie */
- struct qat_sym_nrbg_cookie qsc_nrbg_cookie;
- /* symmetric NRBG cookie */
-#endif
- } u;
+ struct qat_sym_bulk_cookie qsc_bulk_cookie;
/* should be 64-byte aligned */
struct buffer_list_desc qsc_buf_list;
- struct flat_buffer_desc qsc_flat_bufs[QAT_MAXSEG]; /* should be here */
+ struct buffer_list_desc qsc_obuf_list;
- bus_dmamap_t qsc_self_dmamap; /* self DMA mapping and
- end of DMA region */
+ bus_dmamap_t qsc_self_dmamap;
bus_dma_tag_t qsc_self_dma_tag;
uint8_t qsc_iv_buf[EALG_MAX_BLOCK_LEN];
@@ -565,12 +564,11 @@ struct qat_sym_cookie {
uint8_t qsc_gcm_aad[QAT_GCM_AAD_SIZE_MAX];
uint8_t qsc_content_desc[CONTENT_DESC_MAX_SIZE];
- bus_dmamap_t qsc_buf_dmamap; /* qsc_flat_bufs DMA mapping */
- bus_dma_tag_t qsc_buf_dma_tag;
- void *qsc_buf;
+ struct qat_sym_dmamap qsc_dma[QAT_SYM_DMA_COUNT];
bus_addr_t qsc_bulk_req_params_buf_paddr;
bus_addr_t qsc_buffer_list_desc_paddr;
+ bus_addr_t qsc_obuffer_list_desc_paddr;
bus_addr_t qsc_iv_buf_paddr;
bus_addr_t qsc_auth_res_paddr;
bus_addr_t qsc_gcm_aad_paddr;
@@ -578,7 +576,7 @@ struct qat_sym_cookie {
};
CTASSERT(offsetof(struct qat_sym_cookie,
- u.qsc_bulk_cookie.qsbc_req_params_buf) % QAT_OPTIMAL_ALIGN == 0);
+ qsc_bulk_cookie.qsbc_req_params_buf) % QAT_OPTIMAL_ALIGN == 0);
CTASSERT(offsetof(struct qat_sym_cookie, qsc_buf_list) % QAT_OPTIMAL_ALIGN ==
0);
#define MAX_CIPHER_SETUP_BLK_SZ
\
@@ -614,7 +612,6 @@ struct qat_crypto_desc {
uint8_t qcd_req_cache[QAT_MSG_SIZE_TO_BYTES(QAT_MAX_MSG_SIZE)];
} __aligned(QAT_OPTIMAL_ALIGN);
-/* should be aligned to 64bytes */
struct qat_session {
struct qat_crypto_desc *qs_dec_desc; /* should be at top of struct*/
/* decrypt or auth then decrypt or auth */
_______________________________________________
dev-commits-src-main@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/dev-commits-src-main
To unsubscribe, send any mail to "dev-commits-src-main-unsubscr...@freebsd.org"
