Register ahash algorithms for cSHAKE128, cSHAKE256, KMAC128, and KMAC256 using the CMH hash core. cSHAKE supports incremental update and export/import. KMAC has a 64KB data cap imposed by the hardware.
Signed-off-by: Alex Ousherovitch <[email protected]> Co-developed-by: Saravanakrishnan Krishnamoorthy <[email protected]> Signed-off-by: Saravanakrishnan Krishnamoorthy <[email protected]> --- drivers/crypto/cmh/Makefile | 4 +- drivers/crypto/cmh/cmh_cshake.c | 956 ++++++++++++++++++++++++ drivers/crypto/cmh/cmh_kmac.c | 642 ++++++++++++++++ drivers/crypto/cmh/cmh_main.c | 18 + drivers/crypto/cmh/include/cmh_cshake.h | 16 + drivers/crypto/cmh/include/cmh_kmac.h | 16 + 6 files changed, 1651 insertions(+), 1 deletion(-) create mode 100644 drivers/crypto/cmh/cmh_cshake.c create mode 100644 drivers/crypto/cmh/cmh_kmac.c create mode 100644 drivers/crypto/cmh/include/cmh_cshake.h create mode 100644 drivers/crypto/cmh/include/cmh_kmac.h diff --git a/drivers/crypto/cmh/Makefile b/drivers/crypto/cmh/Makefile index acd1827cc084..3ab154744658 100644 --- a/drivers/crypto/cmh/Makefile +++ b/drivers/crypto/cmh/Makefile @@ -16,7 +16,9 @@ cmh-y := \ cmh_key.o \ cmh_sys.o \ cmh_hash.o \ - cmh_hmac.o + cmh_hmac.o \ + cmh_cshake.o \ + cmh_kmac.o # Management ioctl device (/dev/cmh_mgmt): key lifecycle, PKE, PQC ioctls. cmh-$(CONFIG_CRYPTO_DEV_CMH_MGMT) += \ diff --git a/drivers/crypto/cmh/cmh_cshake.c b/drivers/crypto/cmh/cmh_cshake.c new file mode 100644 index 000000000000..3c1712731929 --- /dev/null +++ b/drivers/crypto/cmh/cmh_cshake.c @@ -0,0 +1,956 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- Kernel Crypto API CSHAKE Driver + * + * Registers cSHAKE-128 and cSHAKE-256 as ahash algorithms using the + * CMH Hash Core (HC) via HC_CMD_CSHAKE. + * + * CSHAKE (NIST SP 800-185) extends SHAKE with two domain separation + * parameters: function name N and customization string S. When both + * are empty, cSHAKE reduces to plain SHAKE -- the driver falls back to + * HC_CMD_INIT in that case (per SP 800-185 S6.2). + * + * N and S are set via .setkey() using a self-describing binary header + * (matching the upstream authenc precedent): + * + * struct cshake_cfg { __be32 n_len; __be32 s_len; }; + * setkey blob: cshake_cfg || N[n_len] || S[s_len] + * + * If .setkey() is never called, the driver defaults to plain SHAKE + * (N="" S=""). .setkey() is per-tfm, not per-request. + * + * N is embedded inline in the HC_CMD_CSHAKE struct (max 36 bytes). + * S is passed as VCQ inline data following the command slot (multi-span). + * + * Uses the same streaming transaction model as cmh_hash.c: full sponge + * rate-blocks are absorbed by hardware per .update(), only a sub-rate + * tail is held back, and the running state lives in the SAVEd HC + * context -- so cSHAKE hashes and clones at any length with bounded + * kernel memory. + * .init() -> software-only + * .update() -> [first: CSHAKE(+S) | resume: INIT + RESTORE] + + * UPDATE(full rate-blocks) + SAVE + FLUSH; tail held back + * .final() -> [first: CSHAKE(+S) | resume: INIT + RESTORE] + * [+ UPDATE(residual)] + FINAL + FLUSH + * .export() -> serialise checkpoint + holdback (software-only) + * .import() -> restore checkpoint + holdback (software-only) + * + * The cSHAKE prefix (function name N, customization string S) is + * absorbed once by HC_CMD_CSHAKE on the first submission (bytepad-ded to + * a full rate block, so message absorption stays rate-aligned) and is + * carried thereafter in the SAVEd sponge state; resume re-establishes + * the mode with HC_CMD_INIT before HC_CMD_RESTORE. The HC core supports + * HC_CMD_SAVE / HC_CMD_RESTORE for SHAKE/cSHAKE (outlen stays 0, unlike + * KMAC), which is what enables both streaming and transform cloning. + * + * .setkey() here configures public domain-separation parameters (N, S), + * not a secret key. + */ + +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/crypto.h> +#include <crypto/internal/hash.h> +#include <crypto/scatterwalk.h> +#include <linux/scatterlist.h> +#include <linux/slab.h> +#include <linux/string.h> +#include <asm/byteorder.h> + +#include "cmh_cshake.h" +#include "cmh_vcq.h" +#include "cmh_hc_abi.h" +#include "cmh_txn.h" +#include "cmh_dma.h" + +/* Algorithm Table */ + +/* + * SHAKE sponge rates (SP 800-185): the block granularity for incremental + * absorption. cSHAKE shares the SHAKE rate. + */ +#define CMH_SHAKE128_RATE 168U +#define CMH_SHAKE256_RATE 136U + +struct cmh_cshake_alg_info { + u32 hc_algo; + u32 digest_size; + u32 block_size; /* sponge rate for incremental UPDATE */ + const char *alg_name; + const char *drv_name; +}; + +static const struct cmh_cshake_alg_info cmh_cshake_algs_info[] = { + { + .hc_algo = HC_ALGO_SHAKE128, + .digest_size = CMH_SHAKE128_DIGEST_SIZE, + .block_size = CMH_SHAKE128_RATE, + .alg_name = "cshake128", + .drv_name = "rambus-cmh-cshake128", + }, + { + .hc_algo = HC_ALGO_SHAKE256, + .digest_size = CMH_SHAKE256_DIGEST_SIZE, + .block_size = CMH_SHAKE256_RATE, + .alg_name = "cshake256", + .drv_name = "rambus-cmh-cshake256", + }, +}; + +#define CMH_CSHAKE_ALG_COUNT ARRAY_SIZE(cmh_cshake_algs_info) + +/* Per-Request State */ + +/* Larger of the two sponge rates; sizes the sub-rate holdback buffer. */ +#define CMH_CSHAKE_MAX_RATE CMH_SHAKE128_RATE + +/* + * Max payload slots for a streaming cSHAKE transaction. Worst case is a + * resume with the largest customization string: + * CSHAKE (1) + inline S (4) + RESTORE (1) + UPDATE (1) + SAVE/FINAL (1) + * + FLUSH (1) = 9 + */ +#define CMH_CSHAKE_MAX_PAYLOAD 9 +#define CMH_CSHAKE_MAX_PACKED (CMH_CSHAKE_MAX_PAYLOAD * 2) + +/* Fail the build if a large customization string would overflow cmds[]. */ +static_assert(CMH_CSHAKE_MAX_PAYLOAD >= + 1 + DIV_ROUND_UP(HC_CSHAKE_MAX_CUSTOMLEN, + sizeof(struct vcq_cmd)) + 4); + +/* + * Exported request state (statesize): the HC checkpoint from the last + * SAVE plus the sub-rate holdback. Mirrors the unkeyed hash driver so + * cSHAKE streams to hardware and clones at any length. + */ +struct cmh_cshake_export_state { + u8 checkpoint[HC_CONTEXT_SIZE]; + u8 buf[CMH_CSHAKE_MAX_RATE]; + u32 buf_len; + u32 hw_started; +}; + +/* + * Stored in ahash_request_ctx(). The checkpoint and holdback are + * embedded inline (not heap): the kernel ahash API has no per-request + * destructor, so an abandoned request must not leak. + */ +struct cmh_cshake_reqctx { + const struct cmh_cshake_alg_info *info; + int error; + u32 hw_started; /* non-zero after first HW submission */ + u32 buf_len; /* bytes in holdback buf[] */ + u32 has_checkpoint; /* non-zero if checkpoint[] valid */ + /* DMA state for the current async operation */ + dma_addr_t ckpt_dma; /* RESTORE input */ + dma_addr_t save_dma; /* SAVE output (update only) */ + dma_addr_t data_dma; /* UPDATE input */ + dma_addr_t digest_dma; /* FINAL output (final/digest only) */ + u8 *save_buf; + u8 *data_buf; + u32 data_len; + u8 *digest_buf; + u8 buf[CMH_CSHAKE_MAX_RATE]; /* holdback for partial block */ + u8 checkpoint[HC_CONTEXT_SIZE]; /* HC context from last SAVE */ + struct vcq_cmd packed[CMH_CSHAKE_MAX_PACKED]; +}; + +/* Per-Transform State (carries N and S across requests) */ + +struct cmh_cshake_tfm_ctx { + u8 *func_name; /* N (function name), NULL if empty */ + u32 func_name_len; + u8 *custom; /* S (customization string), NULL if empty */ + u32 custom_len; +}; + +/* VCQ Builders */ + +/* VCQ Builders (cSHAKE-specific; shared builders in cmh_hc_abi.h / cmh_vcq.h) */ + +static void vcq_add_hc_save(struct vcq_cmd *slot, u32 core_id, + u64 output_phys, u32 outlen) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, HC_CMD_SAVE); + slot->hwc.hc.cmd_save.output = output_phys; + slot->hwc.hc.cmd_save.outlen = outlen; +} + +static void vcq_add_hc_restore(struct vcq_cmd *slot, u32 core_id, + u64 input_phys, u32 inlen) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, HC_CMD_RESTORE); + slot->hwc.hc.cmd_restore.input = input_phys; + slot->hwc.hc.cmd_restore.inlen = inlen; +} + +static void vcq_add_hc_cshake(struct vcq_cmd *slot, u32 core_id, u32 algo, + const u8 *name, u32 namelen, + u32 customlen) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, HC_CMD_CSHAKE); + slot->hwc.hc.cmd_cshake.custom = 0; /* inline -- CMH eSW reads from next slot(s) */ + slot->hwc.hc.cmd_cshake.customlen = customlen; + slot->hwc.hc.cmd_cshake.algo = algo; + slot->hwc.hc.cmd_cshake.namelen = namelen; + if (namelen > 0 && name) + memcpy(slot->hwc.hc.cmd_cshake.name, name, + min_t(u32, namelen, HC_CSHAKE_MAX_NAMELEN)); +} + +/* Add an HC_CMD_UPDATE entry */ +static void vcq_add_hc_update(struct vcq_cmd *slot, u32 core_id, u64 input_phys, u32 len) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, HC_CMD_UPDATE); + slot->hwc.hc.cmd_update.input = input_phys; + slot->hwc.hc.cmd_update.inlen = len; +} + +/* + * Emit the HC prologue that (re-)establishes the sponge mode before an + * UPDATE and the trailing SAVE/FINAL: + * cSHAKE (N/S set): CSHAKE [+ inline S] + * plain SHAKE: INIT + * followed by RESTORE when resuming a saved context. The mode command + * is re-issued on resume too: the eSW RESTORE rejects an algo_mode + * mismatch, and cSHAKE uses a distinct mode from INIT. RESTORE then + * overwrites the freshly re-absorbed prefix with the saved sponge state. + * Returns the next free slot index. + */ +static u32 cmh_cshake_emit_prologue(struct vcq_cmd *cmds, u32 idx, + const struct core_dispatch *d, + const struct cmh_cshake_reqctx *rctx, + const struct cmh_cshake_tfm_ctx *tctx, + dma_addr_t ckpt_dma) +{ + if (tctx->func_name_len > 0 || tctx->custom_len > 0) { + u32 span; + + vcq_add_hc_cshake(&cmds[idx], d->core_id, rctx->info->hc_algo, + tctx->func_name, tctx->func_name_len, + tctx->custom_len); + span = vcq_add_inline_data(&cmds[idx], tctx->custom, + tctx->custom_len); + idx += span; + } else { + vcq_add_hc_init(&cmds[idx++], d->core_id, rctx->info->hc_algo); + } + + if (rctx->has_checkpoint) + vcq_add_hc_restore(&cmds[idx++], d->core_id, (u64)ckpt_dma, + HC_CONTEXT_SIZE); + + return idx; +} + +/* Reset per-request state (no heap held between operations). */ +static void cmh_cshake_free_reqctx(struct cmh_cshake_reqctx *rctx) +{ + rctx->has_checkpoint = 0; +} + +/* VCQ Packing + Submit */ + +/* ahash Operations */ + +struct cmh_cshake_alg_drv { + struct ahash_alg alg; + const struct cmh_cshake_alg_info *info; +}; + +static const struct cmh_cshake_alg_info * +cmh_cshake_get_info(struct crypto_ahash *tfm) +{ + struct ahash_alg *alg = crypto_ahash_alg(tfm); + + return container_of(alg, struct cmh_cshake_alg_drv, alg)->info; +} + +/* + * .setkey() -- parse N and S from the self-describing cshake_cfg header. + * + * Blob format: cshake_cfg { __be32 n_len; __be32 s_len; } || N || S + * If never called, the driver defaults to plain SHAKE (N="" S=""). + */ +struct cshake_cfg { + __be32 n_len; + __be32 s_len; +}; + +static int cmh_cshake_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int keylen) +{ + struct cmh_cshake_tfm_ctx *tctx = crypto_ahash_ctx(tfm); + struct cshake_cfg cfg; + u32 n_len, s_len; + const u8 *ptr; + + if (keylen < sizeof(cfg)) + return -EINVAL; + + memcpy(&cfg, key, sizeof(cfg)); + n_len = be32_to_cpu(cfg.n_len); + s_len = be32_to_cpu(cfg.s_len); + + if (keylen != sizeof(cfg) + n_len + s_len) + return -EINVAL; + + if (n_len > HC_CSHAKE_MAX_NAMELEN) + return -EINVAL; + + if (s_len > HC_CSHAKE_MAX_CUSTOMLEN) + return -EINVAL; + + /* + * Free previous N and S. Unlocked against a concurrent .update()/ + * .final() that reads them: the crypto API does not issue setkey + * concurrently with an in-flight request on the same tfm, and the + * only frontend that allowed that race (AF_ALG setsockopt vs I/O) + * is not built on this kernel. + */ + kfree(tctx->func_name); + kfree(tctx->custom); + tctx->func_name = NULL; + tctx->func_name_len = 0; + tctx->custom = NULL; + tctx->custom_len = 0; + + ptr = key + sizeof(cfg); + + if (n_len > 0) { + tctx->func_name = kmemdup(ptr, n_len, GFP_KERNEL); + if (!tctx->func_name) + return -ENOMEM; + tctx->func_name_len = n_len; + ptr += n_len; + } + + if (s_len > 0) { + tctx->custom = kmemdup(ptr, s_len, GFP_KERNEL); + if (!tctx->custom) { + kfree(tctx->func_name); + tctx->func_name = NULL; + tctx->func_name_len = 0; + return -ENOMEM; + } + tctx->custom_len = s_len; + } + + return 0; +} + +static int cmh_cshake_init(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cmh_cshake_reqctx *rctx = ahash_request_ctx(req); + + memset(rctx, 0, sizeof(*rctx)); + rctx->info = cmh_cshake_get_info(tfm); + + return 0; +} + +/* + * Update completion -- runs from the threaded IRQ after SAVE. Takes the + * SAVEd context as the new checkpoint. + */ +static void cmh_cshake_update_complete(void *data, int error) +{ + struct ahash_request *req = data; + struct cmh_cshake_reqctx *rctx = ahash_request_ctx(req); + + if (error == -EINPROGRESS) { + cmh_complete(&req->base, error); + return; + } + + if (rctx->has_checkpoint) + cmh_dma_unmap_single(rctx->ckpt_dma, HC_CONTEXT_SIZE, + DMA_TO_DEVICE); + cmh_dma_unmap_single(rctx->save_dma, HC_CONTEXT_SIZE, + DMA_FROM_DEVICE); + cmh_dma_unmap_single(rctx->data_dma, rctx->data_len, + DMA_TO_DEVICE); + + if (!error) { + memcpy(rctx->checkpoint, rctx->save_buf, HC_CONTEXT_SIZE); + rctx->has_checkpoint = 1; + rctx->hw_started = 1; + } else { + rctx->error = error; + } + + kfree(rctx->save_buf); + rctx->save_buf = NULL; + kfree(rctx->data_buf); + rctx->data_buf = NULL; + rctx->data_len = 0; + + cmh_complete(&req->base, error); +} + +/* + * .update -- absorb full sponge rate-blocks in hardware, hold back the + * sub-rate tail. When at least one full block is available it is + * linearised and submitted as: + * [first: CSHAKE(+S) | resume: INIT + RESTORE] + UPDATE + SAVE + FLUSH + * Returns -EINPROGRESS on HW submission, 0 if only buffering the tail. + */ +static int cmh_cshake_update(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cmh_cshake_tfm_ctx *tctx = crypto_ahash_ctx(tfm); + struct cmh_cshake_reqctx *rctx = ahash_request_ctx(req); + const struct cmh_cshake_alg_info *info = rctx->info; + struct vcq_cmd cmds[CMH_CSHAKE_MAX_PAYLOAD]; + struct core_dispatch d; + u32 block_size = info->block_size; + u32 total_avail, full_len, tail_len, from_src; + u32 idx; + int ret; + gfp_t gfp; + + if (rctx->error) + return rctx->error; + + if (!req->nbytes) + return 0; + + /* buf_len + nbytes must not wrap u32 (mirrors the hash driver). */ + if (req->nbytes > U32_MAX - rctx->buf_len) + return -EMSGSIZE; + + gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; + + total_avail = rctx->buf_len + req->nbytes; + + /* Not enough for a full rate-block -- just buffer the tail. */ + if (total_avail < block_size) { + if (req->base.flags & CRYPTO_AHASH_REQ_VIRT) + memcpy(rctx->buf + rctx->buf_len, + req->svirt, req->nbytes); + else + scatterwalk_map_and_copy(rctx->buf + rctx->buf_len, + req->src, 0, + req->nbytes, 0); + rctx->buf_len = total_avail; + return 0; + } + + /* Have at least one full rate-block -- absorb it in hardware. */ + full_len = total_avail - total_avail % block_size; + tail_len = total_avail - full_len; + from_src = full_len - rctx->buf_len; + + rctx->data_buf = kmalloc(full_len, gfp | __GFP_NOWARN); + if (!rctx->data_buf) + return -ENOMEM; + + if (rctx->buf_len > 0) + memcpy(rctx->data_buf, rctx->buf, rctx->buf_len); + + if (from_src > 0) { + if (req->base.flags & CRYPTO_AHASH_REQ_VIRT) + memcpy(rctx->data_buf + rctx->buf_len, + req->svirt, from_src); + else + scatterwalk_map_and_copy(rctx->data_buf + rctx->buf_len, + req->src, 0, + from_src, 0); + } + + rctx->save_buf = kzalloc(HC_CONTEXT_SIZE, gfp); + if (!rctx->save_buf) { + ret = -ENOMEM; + goto err_free; + } + + rctx->data_dma = cmh_dma_map_single(rctx->data_buf, full_len, + DMA_TO_DEVICE); + if (cmh_dma_map_error(rctx->data_dma)) { + ret = -ENOMEM; + goto err_free; + } + + rctx->save_dma = cmh_dma_map_single(rctx->save_buf, HC_CONTEXT_SIZE, + DMA_FROM_DEVICE); + if (cmh_dma_map_error(rctx->save_dma)) { + ret = -ENOMEM; + goto err_unmap_data; + } + + rctx->ckpt_dma = DMA_MAPPING_ERROR; + if (rctx->has_checkpoint) { + rctx->ckpt_dma = cmh_dma_map_single(rctx->checkpoint, + HC_CONTEXT_SIZE, + DMA_TO_DEVICE); + if (cmh_dma_map_error(rctx->ckpt_dma)) { + ret = -ENOMEM; + goto err_unmap_save; + } + } + + rctx->data_len = full_len; + + d = cmh_core_select_instance(CMH_CORE_HC); + idx = cmh_cshake_emit_prologue(cmds, 0, &d, rctx, tctx, + rctx->ckpt_dma); + + vcq_add_hc_update(&cmds[idx++], d.core_id, + (u64)rctx->data_dma, full_len); + vcq_add_hc_save(&cmds[idx++], d.core_id, + (u64)rctx->save_dma, HC_CONTEXT_SIZE); + vcq_add_flush(&cmds[idx++], d.core_id); + + ret = cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed, + CMH_CSHAKE_MAX_PACKED, + d.mbx_idx, + cmh_cshake_update_complete, req, + !!(req->base.flags & + CRYPTO_TFM_REQ_MAY_BACKLOG), + cmh_tm_async_timeout_jiffies()); + if (ret && ret != -EBUSY) + goto err_unmap_ckpt; + + /* + * Submit accepted (in flight or backlogged) -- only now move the tail + * into the holdback. A synchronous rejection above (e.g. -EAGAIN) + * leaves rctx->buf/buf_len untouched so the caller can retry the + * identical update. + */ + if (tail_len > 0) { + if (req->base.flags & CRYPTO_AHASH_REQ_VIRT) + memcpy(rctx->buf, req->svirt + from_src, tail_len); + else + scatterwalk_map_and_copy(rctx->buf, req->src, + from_src, tail_len, 0); + } + rctx->buf_len = tail_len; + + if (ret == -EBUSY) + return -EBUSY; + return -EINPROGRESS; + +err_unmap_ckpt: + if (rctx->has_checkpoint) + cmh_dma_unmap_single(rctx->ckpt_dma, HC_CONTEXT_SIZE, + DMA_TO_DEVICE); +err_unmap_save: + cmh_dma_unmap_single(rctx->save_dma, HC_CONTEXT_SIZE, + DMA_FROM_DEVICE); +err_unmap_data: + cmh_dma_unmap_single(rctx->data_dma, full_len, DMA_TO_DEVICE); +err_free: + kfree(rctx->save_buf); + rctx->save_buf = NULL; + kfree(rctx->data_buf); + rctx->data_buf = NULL; + rctx->data_len = 0; + return ret; +} + +/* + * Final completion -- unmap DMA, copy digest, signal done. + */ +static void cmh_cshake_final_complete(void *data, int error) +{ + struct ahash_request *req = data; + struct cmh_cshake_reqctx *rctx = ahash_request_ctx(req); + + if (error == -EINPROGRESS) { + cmh_complete(&req->base, error); + return; + } + + if (rctx->has_checkpoint) + cmh_dma_unmap_single(rctx->ckpt_dma, HC_CONTEXT_SIZE, + DMA_TO_DEVICE); + if (rctx->data_buf) + cmh_dma_unmap_single(rctx->data_dma, rctx->data_len, + DMA_TO_DEVICE); + cmh_dma_unmap_single(rctx->digest_dma, rctx->info->digest_size, + DMA_FROM_DEVICE); + + if (!error) + memcpy(req->result, rctx->digest_buf, + rctx->info->digest_size); + + kfree(rctx->digest_buf); + rctx->digest_buf = NULL; + kfree(rctx->data_buf); + rctx->data_buf = NULL; + cmh_cshake_free_reqctx(rctx); + cmh_complete(&req->base, error); +} + +/* + * Submit the final transaction: + * [first: CSHAKE(+S) | resume: INIT + RESTORE] [+ UPDATE(residual)] + * + FINAL + FLUSH + * + * @data_buf: linearised residual (holdback), or NULL for empty input. + * Ownership transferred -- the callback frees it. + */ +static int cmh_cshake_submit_final(struct ahash_request *req, + u8 *data_buf, u32 data_len) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cmh_cshake_tfm_ctx *tctx = crypto_ahash_ctx(tfm); + struct cmh_cshake_reqctx *rctx = ahash_request_ctx(req); + const struct cmh_cshake_alg_info *info = rctx->info; + struct vcq_cmd cmds[CMH_CSHAKE_MAX_PAYLOAD]; + struct core_dispatch d; + u32 idx; + int ret; + gfp_t gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; + + rctx->data_buf = data_buf; + rctx->data_len = data_len; + + rctx->digest_buf = kzalloc(info->digest_size, gfp); + if (!rctx->digest_buf) { + ret = -ENOMEM; + goto err_free_data; + } + + rctx->digest_dma = cmh_dma_map_single(rctx->digest_buf, + info->digest_size, + DMA_FROM_DEVICE); + if (cmh_dma_map_error(rctx->digest_dma)) { + ret = -ENOMEM; + goto err_free_digest; + } + + rctx->data_dma = DMA_MAPPING_ERROR; + if (data_buf && data_len > 0) { + rctx->data_dma = cmh_dma_map_single(data_buf, data_len, + DMA_TO_DEVICE); + if (cmh_dma_map_error(rctx->data_dma)) { + ret = -ENOMEM; + goto err_unmap_digest; + } + } + + rctx->ckpt_dma = DMA_MAPPING_ERROR; + if (rctx->has_checkpoint) { + rctx->ckpt_dma = cmh_dma_map_single(rctx->checkpoint, + HC_CONTEXT_SIZE, + DMA_TO_DEVICE); + if (cmh_dma_map_error(rctx->ckpt_dma)) { + ret = -ENOMEM; + goto err_unmap_data; + } + } + + d = cmh_core_select_instance(CMH_CORE_HC); + idx = cmh_cshake_emit_prologue(cmds, 0, &d, rctx, tctx, + rctx->ckpt_dma); + + if (data_buf && data_len > 0) + vcq_add_hc_update(&cmds[idx++], d.core_id, + (u64)rctx->data_dma, data_len); + + vcq_add_hc_final(&cmds[idx++], d.core_id, + (u64)rctx->digest_dma, info->digest_size); + vcq_add_flush(&cmds[idx++], d.core_id); + + ret = cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed, + CMH_CSHAKE_MAX_PACKED, + d.mbx_idx, + cmh_cshake_final_complete, req, + !!(req->base.flags & + CRYPTO_TFM_REQ_MAY_BACKLOG), + cmh_tm_async_timeout_jiffies()); + if (ret == -EBUSY) + return -EBUSY; + if (ret) + goto err_unmap_ckpt; + + return -EINPROGRESS; + +err_unmap_ckpt: + if (rctx->has_checkpoint) + cmh_dma_unmap_single(rctx->ckpt_dma, HC_CONTEXT_SIZE, + DMA_TO_DEVICE); +err_unmap_data: + if (data_buf && data_len > 0) + cmh_dma_unmap_single(rctx->data_dma, data_len, + DMA_TO_DEVICE); +err_unmap_digest: + cmh_dma_unmap_single(rctx->digest_dma, info->digest_size, + DMA_FROM_DEVICE); +err_free_digest: + kfree(rctx->digest_buf); + rctx->digest_buf = NULL; +err_free_data: + kfree(data_buf); + rctx->data_buf = NULL; + cmh_cshake_free_reqctx(rctx); + return ret; +} + +static int cmh_cshake_final(struct ahash_request *req) +{ + struct cmh_cshake_reqctx *rctx = ahash_request_ctx(req); + u8 *data_buf = NULL; + u32 data_len = 0; + gfp_t gfp; + + if (rctx->error) + return rctx->error; + + if (rctx->buf_len > 0) { + gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; + data_buf = kmalloc(rctx->buf_len, gfp); + if (!data_buf) + return -ENOMEM; + memcpy(data_buf, rctx->buf, rctx->buf_len); + data_len = rctx->buf_len; + rctx->buf_len = 0; + } + + return cmh_cshake_submit_final(req, data_buf, data_len); +} + +/* + * .finup -- update + final in one transaction. Linearises the holdback + * and new data and submits through the final path (avoids the kernel's + * ahash_def_finup() clone-via-export path). + */ +static int cmh_cshake_finup(struct ahash_request *req) +{ + struct cmh_cshake_reqctx *rctx = ahash_request_ctx(req); + u32 data_len; + u8 *data_buf; + gfp_t gfp; + + if (rctx->error) + return rctx->error; + + if (req->nbytes > U32_MAX - rctx->buf_len) + return -EMSGSIZE; + + data_len = rctx->buf_len + req->nbytes; + + if (data_len == 0) + return cmh_cshake_submit_final(req, NULL, 0); + + gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; + + data_buf = kmalloc(data_len, gfp | __GFP_NOWARN); + if (!data_buf) + return -ENOMEM; + + if (rctx->buf_len > 0) + memcpy(data_buf, rctx->buf, rctx->buf_len); + + if (req->nbytes > 0) { + if (req->base.flags & CRYPTO_AHASH_REQ_VIRT) + memcpy(data_buf + rctx->buf_len, + req->svirt, req->nbytes); + else + scatterwalk_map_and_copy(data_buf + rctx->buf_len, + req->src, 0, + req->nbytes, 0); + } + + rctx->buf_len = 0; + return cmh_cshake_submit_final(req, data_buf, data_len); +} + +static int cmh_cshake_digest(struct ahash_request *req) +{ + int ret; + + ret = cmh_cshake_init(req); + if (ret) + return ret; + + return cmh_cshake_finup(req); +} + +/* + * Export -- purely software. Serialise the HC checkpoint (if any) and + * the holdback into the export state; the streaming .update() keeps the + * checkpoint current after each block. + */ +static int cmh_cshake_export(struct ahash_request *req, void *out) +{ + struct cmh_cshake_reqctx *rctx = ahash_request_ctx(req); + struct cmh_cshake_export_state *state = out; + + /* + * Zero the whole state first: only buf_len bytes of buf[] are + * populated and the struct may carry padding, so this avoids + * leaking kernel memory through the ahash export. + */ + memset(state, 0, sizeof(*state)); + + if (rctx->hw_started) + memcpy(state->checkpoint, rctx->checkpoint, HC_CONTEXT_SIZE); + + if (rctx->buf_len > 0) + memcpy(state->buf, rctx->buf, rctx->buf_len); + + state->buf_len = rctx->buf_len; + state->hw_started = rctx->hw_started; + + return 0; +} + +/* + * Import -- purely software. Restore checkpoint and holdback; the next + * .update()/.final() RESTOREs the checkpoint into HW. + */ +static int cmh_cshake_import(struct ahash_request *req, const void *in) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cmh_cshake_reqctx *rctx = ahash_request_ctx(req); + const struct cmh_cshake_export_state *state = in; + + memset(rctx, 0, sizeof(*rctx)); + rctx->info = cmh_cshake_get_info(tfm); + + /* + * The holdback invariant is buf_len < block_size (update() never + * leaves a full block held back); a larger value would overrun the + * linearisation buffer in the next update(). + */ + if (state->buf_len >= rctx->info->block_size) + return -EINVAL; + + rctx->hw_started = state->hw_started; + rctx->buf_len = state->buf_len; + memcpy(rctx->buf, state->buf, state->buf_len); + + if (state->hw_started) { + memcpy(rctx->checkpoint, state->checkpoint, HC_CONTEXT_SIZE); + rctx->has_checkpoint = 1; + } + + return 0; +} + +/* Transform init/exit */ + +static int cmh_cshake_cra_init(struct crypto_tfm *tfm) +{ + struct cmh_cshake_tfm_ctx *tctx = crypto_tfm_ctx(tfm); + + tctx->func_name = NULL; + tctx->func_name_len = 0; + tctx->custom = NULL; + tctx->custom_len = 0; + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct cmh_cshake_reqctx)); + return 0; +} + +static void cmh_cshake_cra_exit(struct crypto_tfm *tfm) +{ + struct cmh_cshake_tfm_ctx *tctx = crypto_tfm_ctx(tfm); + + kfree(tctx->func_name); + kfree(tctx->custom); + tctx->func_name = NULL; + tctx->custom = NULL; +} + +/* Registration */ + +static struct cmh_cshake_alg_drv cmh_cshake_drvs[CMH_CSHAKE_ALG_COUNT]; + +/** + * cmh_cshake_register() - Register cSHAKE-128/256 hash algorithms with the crypto framework + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_cshake_register(void) +{ + unsigned int i; + int ret; + + if (!cmh_core_present(CMH_CORE_HC)) + return 0; + + for (i = 0; i < CMH_CSHAKE_ALG_COUNT; i++) { + const struct cmh_cshake_alg_info *info = + &cmh_cshake_algs_info[i]; + struct cmh_cshake_alg_drv *drv = &cmh_cshake_drvs[i]; + struct ahash_alg *alg = &drv->alg; + + drv->info = info; + + alg->init = cmh_cshake_init; + alg->update = cmh_cshake_update; + alg->final = cmh_cshake_final; + alg->finup = cmh_cshake_finup; + alg->digest = cmh_cshake_digest; + alg->export = cmh_cshake_export; + alg->import = cmh_cshake_import; + alg->setkey = cmh_cshake_setkey; + + alg->halg.digestsize = info->digest_size; + alg->halg.statesize = sizeof(struct cmh_cshake_export_state); + + strscpy(alg->halg.base.cra_name, info->alg_name, + CRYPTO_MAX_ALG_NAME); + strscpy(alg->halg.base.cra_driver_name, info->drv_name, + CRYPTO_MAX_ALG_NAME); + alg->halg.base.cra_priority = 300; + alg->halg.base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NO_FALLBACK | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_OPTIONAL_KEY | + CRYPTO_ALG_REQ_VIRT; + alg->halg.base.cra_blocksize = 1; /* XOF */ + alg->halg.base.cra_ctxsize = sizeof(struct cmh_cshake_tfm_ctx); + alg->halg.base.cra_init = cmh_cshake_cra_init; + alg->halg.base.cra_exit = cmh_cshake_cra_exit; + alg->halg.base.cra_module = THIS_MODULE; + + ret = crypto_register_ahash(alg); + if (ret) { + dev_err(cmh_dev(), "cshake: failed to register %s (rc=%d)\n", + info->drv_name, ret); + while (i--) + crypto_unregister_ahash(&cmh_cshake_drvs[i].alg); + return ret; + } + + dev_dbg(cmh_dev(), "cshake: registered %s (priority 300)\n", + info->drv_name); + } + + return 0; +} + +/** + * cmh_cshake_unregister() - Unregister cSHAKE hash algorithms from the crypto framework + */ +void cmh_cshake_unregister(void) +{ + unsigned int i; + + if (!cmh_core_present(CMH_CORE_HC)) + return; + + for (i = 0; i < CMH_CSHAKE_ALG_COUNT; i++) { + crypto_unregister_ahash(&cmh_cshake_drvs[i].alg); + dev_dbg(cmh_dev(), "cshake: unregistered %s\n", + cmh_cshake_algs_info[i].drv_name); + } +} diff --git a/drivers/crypto/cmh/cmh_kmac.c b/drivers/crypto/cmh/cmh_kmac.c new file mode 100644 index 000000000000..22eaf8975a26 --- /dev/null +++ b/drivers/crypto/cmh/cmh_kmac.c @@ -0,0 +1,642 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- Kernel Crypto API KMAC Driver + * + * Registers KMAC-128 and KMAC-256 as keyed ahash algorithms using the + * CMH Hash Core (HC) via HC_CMD_KMAC. + * + * KMAC (NIST SP 800-185) is a keyed variant of cSHAKE. The function + * name N is always "KMAC" (hardcoded by the CMH eSW). The user sets: + * - A key via .setkey() (raw bytes + optional S) + * - An optional customization string S via the setkey blob + * + * setkey blob format: + * struct kmac_key_param { __be32 keylen; __be32 s_len; }; + * blob: kmac_key_param || key[keylen] || S[s_len] + * + * Uses the same self-contained transaction model as cmh_hmac.c: + * .setkey() -> store raw key (+ S) + * .init() -> software-only + * .update() -> software-only (accumulate chunks) + * .final() -> [SYS_CMD_WRITE] + HC_CMD_KMAC [+ inline S] + + * [GATHER] + FINAL + FLUSH + */ + +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/crypto.h> +#include <crypto/internal/hash.h> +#include <linux/scatterlist.h> +#include <linux/list.h> +#include <linux/slab.h> +#include <linux/string.h> +#include <asm/byteorder.h> + +#include "cmh_kmac.h" +#include "cmh_vcq.h" +#include "cmh_hc_abi.h" +#include "cmh_sys_abi.h" +#include "cmh_sys.h" +#include "cmh_txn.h" +#include "cmh_dma.h" +#include "cmh_key.h" + +/* + * Maximum data that can be accumulated across .update() calls. + * The CMH eSW rejects HC_CMD_SAVE when ctx->outlen != 0, which is + * always the case for KMAC (eip59_hc_kmac() sets ctx->outlen for + * right_encode(outlen) at finalization). All data must be buffered + * in kernel memory and submitted atomically in .final(). + * + * The CMH eSW does not serialize outlen into the external save + * context, so HC_CMD_SAVE fails for KMAC mode. + */ +#define KMAC_MAX_DATA (64 * 1024) + +/* Algorithm Table */ + +struct cmh_kmac_alg_info { + u32 hc_algo; + u32 digest_size; + const char *alg_name; + const char *drv_name; +}; + +static const struct cmh_kmac_alg_info cmh_kmac_algs_info[] = { + { + .hc_algo = HC_ALGO_SHAKE128, + .digest_size = CMH_SHAKE128_DIGEST_SIZE, + .alg_name = "kmac128", + .drv_name = "rambus-cmh-kmac128", + }, + { + .hc_algo = HC_ALGO_SHAKE256, + .digest_size = CMH_SHAKE256_DIGEST_SIZE, + .alg_name = "kmac256", + .drv_name = "rambus-cmh-kmac256", + }, +}; + +#define CMH_KMAC_ALG_COUNT ARRAY_SIZE(cmh_kmac_algs_info) + +/* Per-Request State */ + +struct cmh_kmac_chunk { + struct list_head list; + struct list_head tfm_node; /* per-tfm orphan tracking */ + u32 len; + u8 data[]; +}; + +/* + * Max payload slots for KMAC: + * SYS_CMD_WRITE (1) + KMAC (1) + inline S (3 max) + GATHER (1) + + * FINAL (1) + FLUSH (1) = 8 + */ +#define CMH_KMAC_MAX_PAYLOAD 9 +#define CMH_KMAC_MAX_PACKED (CMH_KMAC_MAX_PAYLOAD * 2) + +struct cmh_kmac_reqctx { + const struct cmh_kmac_alg_info *info; + int error; + struct list_head chunks; + u32 num_chunks; + u32 total_len; + /* DMA state for async final */ + dma_addr_t digest_dma; + dma_addr_t key_dma; + u8 *digest_buf; + struct cmh_sg_map *sgm; + u32 keylen; + struct vcq_cmd packed[CMH_KMAC_MAX_PACKED]; +}; + +/* Per-Transform State (carries key + S across requests) */ + +struct cmh_kmac_tfm_ctx { + struct cmh_key_ctx key; + u8 *custom; /* S (customization string), NULL if empty */ + u32 custom_len; + spinlock_t chunk_lock; /* protects all_chunks */ + struct list_head all_chunks; /* orphan-safe chunk tracking */ +}; + +/* VCQ Builders (KMAC-specific; shared builders in cmh_hc_abi.h / cmh_vcq.h) */ + +static void vcq_add_hc_kmac(struct vcq_cmd *slot, u32 core_id, u64 key_ref, u32 keylen, + u32 customlen, u32 algo, u32 outlen) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, HC_CMD_KMAC); + slot->hwc.hc.cmd_kmac.key = key_ref; + slot->hwc.hc.cmd_kmac.custom = 0; /* inline */ + slot->hwc.hc.cmd_kmac.keylen = keylen; + slot->hwc.hc.cmd_kmac.customlen = customlen; + slot->hwc.hc.cmd_kmac.algo = algo; + slot->hwc.hc.cmd_kmac.outlen = outlen; +} + +/* Request Context Cleanup */ + +static void cmh_kmac_free_chunks(struct cmh_kmac_reqctx *rctx, + struct cmh_kmac_tfm_ctx *tctx) +{ + struct cmh_kmac_chunk *chunk, *tmp; + + spin_lock_bh(&tctx->chunk_lock); + list_for_each_entry_safe(chunk, tmp, &rctx->chunks, list) { + list_del(&chunk->list); + list_del(&chunk->tfm_node); + kfree(chunk); + } + spin_unlock_bh(&tctx->chunk_lock); + rctx->num_chunks = 0; + rctx->total_len = 0; +} + +static struct cmh_sg_map * +cmh_kmac_build_sg(struct cmh_kmac_reqctx *rctx, gfp_t gfp) +{ + struct cmh_dma_buf *bufs; + struct cmh_kmac_chunk *chunk; + struct cmh_sg_map *sgm; + u32 i; + + bufs = kcalloc(rctx->num_chunks, sizeof(*bufs), gfp); + if (!bufs) + return NULL; + + i = 0; + list_for_each_entry(chunk, &rctx->chunks, list) { + bufs[i].data = chunk->data; + bufs[i].len = chunk->len; + i++; + } + + sgm = cmh_dma_build_sg(bufs, rctx->num_chunks, gfp); + kfree(bufs); + return sgm; +} + +/* VCQ Packing + Submit */ + +/* ahash Operations */ + +struct cmh_kmac_alg_drv { + struct ahash_alg alg; + const struct cmh_kmac_alg_info *info; +}; + +static const struct cmh_kmac_alg_info * +cmh_kmac_get_info(struct crypto_ahash *tfm) +{ + struct ahash_alg *alg = crypto_ahash_alg(tfm); + + return container_of(alg, struct cmh_kmac_alg_drv, alg)->info; +} + +/* + * setkey blob for KMAC (raw key path): + * struct kmac_key_param { __be32 keylen; __be32 s_len; }; + * blob: kmac_key_param || key[keylen] || S[s_len] + */ +struct kmac_key_param { + __be32 keylen; + __be32 s_len; +}; + +static int cmh_kmac_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int keylen) +{ + struct cmh_kmac_tfm_ctx *tctx = crypto_ahash_ctx(tfm); + /* raw key bytes with optional S */ + { + struct kmac_key_param hdr; + u32 raw_keylen, s_len; + const u8 *ptr; + + if (keylen < sizeof(hdr)) + return -EINVAL; + + memcpy(&hdr, key, sizeof(hdr)); + raw_keylen = be32_to_cpu(hdr.keylen); + s_len = be32_to_cpu(hdr.s_len); + + if (keylen != sizeof(hdr) + raw_keylen + s_len) + return -EINVAL; + + if (raw_keylen == 0) + return -EINVAL; + + if (s_len > HC_CSHAKE_MAX_CUSTOMLEN) + return -EINVAL; + + ptr = key + sizeof(hdr); + + /* Store raw key */ + { + int ret = cmh_key_setkey_raw(&tctx->key, ptr, + raw_keylen, CORE_ID_HC); + if (ret) + return ret; + } + ptr += raw_keylen; + + /* + * Store S. The raw key (above) and S are replaced without a + * lock against a concurrent .final() reader; safe because the + * crypto API does not issue setkey during an in-flight request + * on the same tfm, and AF_ALG (the only racing frontend) is + * not built on this kernel. + */ + kfree(tctx->custom); + tctx->custom = NULL; + tctx->custom_len = 0; + + if (s_len > 0) { + tctx->custom = kmemdup(ptr, s_len, GFP_KERNEL); + if (!tctx->custom) { + cmh_key_destroy(&tctx->key); + return -ENOMEM; + } + tctx->custom_len = s_len; + } + + return 0; + } +} + +static int cmh_kmac_init(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cmh_kmac_reqctx *rctx = ahash_request_ctx(req); + + rctx->info = cmh_kmac_get_info(tfm); + rctx->error = 0; + INIT_LIST_HEAD(&rctx->chunks); + rctx->num_chunks = 0; + rctx->total_len = 0; + + return 0; +} + +static int cmh_kmac_update(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cmh_kmac_tfm_ctx *tctx = crypto_ahash_ctx(tfm); + struct cmh_kmac_reqctx *rctx = ahash_request_ctx(req); + struct cmh_kmac_chunk *chunk; + int nents; + + if (rctx->error) + return rctx->error; + + if (!req->nbytes) + return 0; + + /* + * Cap cumulative input at KMAC_MAX_DATA (64 KB) before the alloc + * below, so sizeof(*chunk) + req->nbytes cannot overflow size_t. + */ + if (req->nbytes > KMAC_MAX_DATA - rctx->total_len) { + rctx->error = -EINVAL; + goto err_free_chunks; + } + + chunk = kmalloc(sizeof(*chunk) + req->nbytes, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC); + if (!chunk) { + rctx->error = -ENOMEM; + goto err_free_chunks; + } + + chunk->len = req->nbytes; + if (req->base.flags & CRYPTO_AHASH_REQ_VIRT) { + memcpy(chunk->data, req->svirt, req->nbytes); + } else { + nents = sg_nents_for_len(req->src, req->nbytes); + if (nents < 0 || + sg_copy_to_buffer(req->src, nents, + chunk->data, req->nbytes) != req->nbytes) { + kfree(chunk); + rctx->error = -EINVAL; + goto err_free_chunks; + } + } + + list_add_tail(&chunk->list, &rctx->chunks); + spin_lock_bh(&tctx->chunk_lock); + list_add_tail(&chunk->tfm_node, &tctx->all_chunks); + spin_unlock_bh(&tctx->chunk_lock); + rctx->num_chunks++; + rctx->total_len += req->nbytes; + + return 0; + +err_free_chunks: + /* + * Terminal error -- free all previously accumulated chunks. + * The crypto API hash path does not call .final() on error, + * so chunks would be orphaned otherwise. + */ + cmh_kmac_free_chunks(rctx, tctx); + return rctx->error; +} + +static void cmh_kmac_complete(void *data, int error) +{ + struct ahash_request *req = data; + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cmh_kmac_tfm_ctx *tctx = crypto_ahash_ctx(tfm); + struct cmh_kmac_reqctx *rctx = ahash_request_ctx(req); + + if (error == -EINPROGRESS) { + cmh_complete(&req->base, error); + return; + } + + cmh_dma_unmap_single(rctx->digest_dma, rctx->info->digest_size, + DMA_FROM_DEVICE); + + if (!error) + memcpy(req->result, rctx->digest_buf, + rctx->info->digest_size); + + kfree(rctx->digest_buf); + rctx->digest_buf = NULL; + cmh_dma_free_sg(rctx->sgm); + rctx->sgm = NULL; + cmh_kmac_free_chunks(rctx, tctx); + cmh_complete(&req->base, error); +} + +static int cmh_kmac_final(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cmh_kmac_tfm_ctx *tctx = crypto_ahash_ctx(tfm); + struct cmh_kmac_reqctx *rctx = ahash_request_ctx(req); + const struct cmh_kmac_alg_info *info = rctx->info; + struct vcq_cmd cmds[CMH_KMAC_MAX_PAYLOAD]; + struct cmh_sg_map *sgm = NULL; + dma_addr_t digest_dma = DMA_MAPPING_ERROR, key_dma = DMA_MAPPING_ERROR; + u8 *digest_buf; + u64 key_ref; + u32 key_len; + struct core_dispatch d; + s32 target_mbx; + u32 core_id; + u32 idx; + int ret; + gfp_t gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; + + if (rctx->error) { + ret = rctx->error; + goto out_free; + } + + if (tctx->key.mode == CMH_KEY_NONE) { + ret = -ENOKEY; + goto out_free; + } + + if (rctx->num_chunks > 0) { + sgm = cmh_kmac_build_sg(rctx, gfp); + if (!sgm) { + ret = -ENOMEM; + goto out_free; + } + } + + digest_buf = kzalloc(info->digest_size, gfp); + if (!digest_buf) { + ret = -ENOMEM; + goto out_free_sg; + } + digest_dma = cmh_dma_map_single(digest_buf, info->digest_size, + DMA_FROM_DEVICE); + if (cmh_dma_map_error(digest_dma)) { + ret = -ENOMEM; + goto out_free_digest; + } + + /* Resolve key reference */ + idx = 0; + + key_dma = tctx->key.raw.dma; + vcq_add_sys_write(&cmds[idx++], SYS_REF_TEMP, (u64)key_dma, + SYS_REF_NONE, tctx->key.raw.len, + tctx->key.raw.sys_type); + key_ref = SYS_REF_TEMP; + key_len = tctx->key.raw.len; + d = cmh_core_select_instance(CMH_CORE_HC); + + target_mbx = d.mbx_idx; + + core_id = d.core_id; + + { + u32 span; + + vcq_add_hc_kmac(&cmds[idx], core_id, key_ref, key_len, + tctx->custom_len, info->hc_algo, + info->digest_size); + + /* Add inline S data after the KMAC slot */ + span = vcq_add_inline_data(&cmds[idx], tctx->custom, + tctx->custom_len); + idx += span; + } + + if (sgm) + vcq_add_hc_gather(&cmds[idx++], core_id, (u64)sgm->items_dma, + HC_CMD_UPDATE); + + vcq_add_hc_final(&cmds[idx++], core_id, (u64)digest_dma, info->digest_size); + vcq_add_flush(&cmds[idx++], core_id); + + rctx->digest_buf = digest_buf; + rctx->digest_dma = digest_dma; + rctx->sgm = sgm; + + ret = cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed, + CMH_KMAC_MAX_PACKED, + target_mbx, + cmh_kmac_complete, req, + !!(req->base.flags & + CRYPTO_TFM_REQ_MAY_BACKLOG), + cmh_tm_async_timeout_jiffies()); + if (ret == -EBUSY) + return -EBUSY; + if (ret) + goto out_cleanup_all; + + return -EINPROGRESS; + +out_cleanup_all: + cmh_dma_unmap_single(digest_dma, info->digest_size, + DMA_FROM_DEVICE); +out_free_digest: + kfree(digest_buf); + +out_free_sg: + cmh_dma_free_sg(sgm); + +out_free: + cmh_kmac_free_chunks(rctx, tctx); + return ret; +} + +static int cmh_kmac_finup(struct ahash_request *req) +{ + int ret; + + ret = cmh_kmac_update(req); + if (ret) + return ret; + + return cmh_kmac_final(req); +} + +static int cmh_kmac_digest(struct ahash_request *req) +{ + int ret; + + ret = cmh_kmac_init(req); + if (ret) + return ret; + + return cmh_kmac_finup(req); +} + +static int cmh_kmac_export(struct ahash_request *req, void *out) +{ + return -EOPNOTSUPP; +} + +static int cmh_kmac_import(struct ahash_request *req, const void *in) +{ + return -EOPNOTSUPP; +} + +/* Transform init/exit */ + +static int cmh_kmac_cra_init(struct crypto_tfm *tfm) +{ + struct cmh_kmac_tfm_ctx *tctx = crypto_tfm_ctx(tfm); + + tctx->key.mode = CMH_KEY_NONE; + tctx->custom = NULL; + tctx->custom_len = 0; + spin_lock_init(&tctx->chunk_lock); + INIT_LIST_HEAD(&tctx->all_chunks); + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct cmh_kmac_reqctx)); + return 0; +} + +static void cmh_kmac_cra_exit(struct crypto_tfm *tfm) +{ + struct cmh_kmac_tfm_ctx *tctx = crypto_tfm_ctx(tfm); + struct cmh_kmac_chunk *chunk, *tmp; + + /* Free any orphaned chunks (e.g. testmgr export/reimport poison) */ + spin_lock_bh(&tctx->chunk_lock); + list_for_each_entry_safe(chunk, tmp, &tctx->all_chunks, tfm_node) { + list_del(&chunk->tfm_node); + kfree(chunk); + } + spin_unlock_bh(&tctx->chunk_lock); + + cmh_key_destroy(&tctx->key); + kfree(tctx->custom); + tctx->custom = NULL; +} + +/* Registration */ + +static struct cmh_kmac_alg_drv cmh_kmac_drvs[CMH_KMAC_ALG_COUNT]; + +/** + * cmh_kmac_register() - Register KMAC-128/256 hash algorithms with the crypto framework + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_kmac_register(void) +{ + unsigned int i; + int ret; + + if (!cmh_core_present(CMH_CORE_HC)) + return 0; + + for (i = 0; i < CMH_KMAC_ALG_COUNT; i++) { + const struct cmh_kmac_alg_info *info = + &cmh_kmac_algs_info[i]; + struct cmh_kmac_alg_drv *drv = &cmh_kmac_drvs[i]; + struct ahash_alg *alg = &drv->alg; + + drv->info = info; + + alg->init = cmh_kmac_init; + alg->update = cmh_kmac_update; + alg->final = cmh_kmac_final; + alg->finup = cmh_kmac_finup; + alg->digest = cmh_kmac_digest; + alg->export = cmh_kmac_export; + alg->import = cmh_kmac_import; + alg->setkey = cmh_kmac_setkey; + + alg->halg.digestsize = info->digest_size; + alg->halg.statesize = sizeof(struct cmh_kmac_reqctx); + + strscpy(alg->halg.base.cra_name, info->alg_name, + CRYPTO_MAX_ALG_NAME); + strscpy(alg->halg.base.cra_driver_name, info->drv_name, + CRYPTO_MAX_ALG_NAME); + alg->halg.base.cra_priority = 300; + alg->halg.base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NO_FALLBACK | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_REQ_VIRT; + alg->halg.base.cra_blocksize = 1; /* XOF/keyed XOF */ + alg->halg.base.cra_ctxsize = sizeof(struct cmh_kmac_tfm_ctx); + alg->halg.base.cra_init = cmh_kmac_cra_init; + alg->halg.base.cra_exit = cmh_kmac_cra_exit; + alg->halg.base.cra_module = THIS_MODULE; + + ret = crypto_register_ahash(alg); + if (ret) { + dev_err(cmh_dev(), "kmac: failed to register %s (rc=%d)\n", + info->drv_name, ret); + while (i--) + crypto_unregister_ahash(&cmh_kmac_drvs[i].alg); + return ret; + } + + dev_dbg(cmh_dev(), "kmac: registered %s (priority 300)\n", + info->drv_name); + } + + return 0; +} + +/** + * cmh_kmac_unregister() - Unregister KMAC hash algorithms from the crypto framework + */ +void cmh_kmac_unregister(void) +{ + unsigned int i; + + if (!cmh_core_present(CMH_CORE_HC)) + return; + + for (i = 0; i < CMH_KMAC_ALG_COUNT; i++) { + crypto_unregister_ahash(&cmh_kmac_drvs[i].alg); + dev_dbg(cmh_dev(), "kmac: unregistered %s\n", + cmh_kmac_algs_info[i].drv_name); + } +} diff --git a/drivers/crypto/cmh/cmh_main.c b/drivers/crypto/cmh/cmh_main.c index 589363cc4b28..474b438fc18a 100644 --- a/drivers/crypto/cmh/cmh_main.c +++ b/drivers/crypto/cmh/cmh_main.c @@ -33,6 +33,8 @@ #include "cmh_rh.h" #include "cmh_hash.h" #include "cmh_hmac.h" +#include "cmh_cshake.h" +#include "cmh_kmac.h" #include "cmh_mgmt.h" #include "cmh_registers.h" #include "cmh_debugfs.h" @@ -214,6 +216,16 @@ static int cmh_probe(struct platform_device *pdev) if (ret) goto err_hmac_register; + /* Register CSHAKE hash algorithms */ + ret = cmh_cshake_register(); + if (ret) + goto err_cshake_register; + + /* Register KMAC hash algorithms */ + ret = cmh_kmac_register(); + if (ret) + goto err_kmac_register; + /* Register key management device (/dev/cmh_mgmt) */ ret = cmh_mgmt_register(); if (ret) @@ -224,6 +236,10 @@ static int cmh_probe(struct platform_device *pdev) return 0; err_mgmt_register: + cmh_kmac_unregister(); +err_kmac_register: + cmh_cshake_unregister(); +err_cshake_register: cmh_hmac_unregister(); err_hmac_register: cmh_hash_unregister(); @@ -254,6 +270,8 @@ static void cmh_remove(struct platform_device *pdev) cfg = &dev->config; cmh_mgmt_unregister(); + cmh_kmac_unregister(); + cmh_cshake_unregister(); cmh_hmac_unregister(); cmh_hash_unregister(); cmh_rh_cleanup(cfg); diff --git a/drivers/crypto/cmh/include/cmh_cshake.h b/drivers/crypto/cmh/include/cmh_cshake.h new file mode 100644 index 000000000000..9bafe0baf52f --- /dev/null +++ b/drivers/crypto/cmh/include/cmh_cshake.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- Kernel Crypto API CSHAKE Driver + * + * Registers cSHAKE-128 and cSHAKE-256 ahash algorithms using + * HC_CMD_CSHAKE with inline customization string S. + */ + +#ifndef CMH_CSHAKE_H +#define CMH_CSHAKE_H + +int cmh_cshake_register(void); +void cmh_cshake_unregister(void); + +#endif /* CMH_CSHAKE_H */ diff --git a/drivers/crypto/cmh/include/cmh_kmac.h b/drivers/crypto/cmh/include/cmh_kmac.h new file mode 100644 index 000000000000..b3c92d71a0b6 --- /dev/null +++ b/drivers/crypto/cmh/include/cmh_kmac.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- Kernel Crypto API KMAC Driver + * + * Registers KMAC-128 and KMAC-256 ahash algorithms using + * HC_CMD_KMAC with inline customization string S. + */ + +#ifndef CMH_KMAC_H +#define CMH_KMAC_H + +int cmh_kmac_register(void); +void cmh_kmac_unregister(void); + +#endif /* CMH_KMAC_H */ -- 2.43.7
