/* $NetBSD: sun8i_crypto.c,v 1.32 2022/05/22 11:39:26 riastradh Exp $ */
/*-
* Copyright (c) 2019 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
* by Taylor R. Campbell.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
* TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
/*
* sun8i_crypto -- Allwinner Crypto Engine driver
*
* The Crypto Engine is documented in Sec. 3.15 of the Allwinner A64
* User Manual v1.1, on pp. 230--241. We only use it for the TRNG at
* the moment, but in principle it could be wired up with opencrypto(9)
* to compute AES, DES, 3DES, MD5, SHA-1, SHA-224, SHA-256, HMAC-SHA1,
* HMAC-HA256, RSA, and an undocumented PRNG. It also seems to support
* AES keys in SRAM (for some kind of HDMI HDCP stuff?).
*
* https://linux-sunxi.org/images/b/b4/Allwinner_A64_User_Manual_V1.1.pdf
*/
#include <sys/cdefs.h>
__KERNEL_RCSID(1, "$NetBSD: sun8i_crypto.c,v 1.32 2022/05/22 11:39:26 riastradh Exp $");
#include <sys/types.h>
#include <sys/param.h>
#include <sys/atomic.h>
#include <sys/bus.h>
#include <sys/callout.h>
#include <sys/conf.h>
#include <sys/cprng.h>
#include <sys/device.h>
#include <sys/kernel.h>
#include <sys/kmem.h>
#include <sys/mbuf.h>
#include <sys/mutex.h>
#include <sys/rndsource.h>
#include <sys/sdt.h>
#include <sys/sysctl.h>
#include <sys/workqueue.h>
#include <dev/fdt/fdtvar.h>
#include <opencrypto/cryptodev.h>
#include <arm/sunxi/sun8i_crypto.h>
#define SUN8I_CRYPTO_TIMEOUT hz
#define SUN8I_CRYPTO_RNGENTROPY 100 /* estimated bits per bit of entropy */
#define SUN8I_CRYPTO_RNGBYTES PAGE_SIZE
struct sun8i_crypto_config {
u_int mod_rate; /* module clock rate */
};
/*
* The module clock is set to 50 MHz on H3, 300 MHz otherwise.
* From Linux drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c:
* Module clock is lower on H3 than other SoC due to some DMA
* timeout occurring with high value.
*/
static const struct sun8i_crypto_config sun50i_a64_crypto_config = {
.mod_rate = 300*1000*1000,
};
static const struct sun8i_crypto_config sun50i_h5_crypto_config = {
.mod_rate = 300*1000*1000,
};
static const struct sun8i_crypto_config sun8i_h3_crypto_config = {
.mod_rate = 50*1000*1000,
};
struct sun8i_crypto_task;
struct sun8i_crypto_buf {
bus_dma_segment_t cb_seg[1];
int cb_nsegs;
void *cb_kva;
};
struct sun8i_crypto_softc {
device_t sc_dev;
bus_space_tag_t sc_bst;
bus_space_handle_t sc_bsh;
bus_dma_tag_t sc_dmat;
struct pool_cache *sc_taskpool;
const struct sun8i_crypto_config *sc_cfg;
struct workqueue *sc_wq;
void *sc_ih;
bool sc_polling;
kmutex_t sc_lock;
struct sun8i_crypto_chan {
struct sun8i_crypto_task *cc_task;
unsigned cc_starttime;
} sc_chan[SUN8I_CRYPTO_NCHAN];
struct callout sc_timeout;
kmutex_t sc_intr_lock;
uint32_t sc_done;
uint32_t sc_esr;
struct work sc_work;
bool sc_work_pending;
struct sun8i_crypto_rng {
struct sun8i_crypto_buf cr_buf;
struct sun8i_crypto_task *cr_task;
struct krndsource cr_rndsource;
bool cr_pending;
} sc_rng;
struct sun8i_crypto_selftest {
struct sun8i_crypto_buf cs_in;
struct sun8i_crypto_buf cs_key;
struct sun8i_crypto_buf cs_out;
struct sun8i_crypto_task *cs_task;
bool cs_pending;
bool cs_passed;
} sc_selftest;
struct sun8i_crypto_sysctl {
struct sysctllog *cy_log;
const struct sysctlnode *cy_root_node;
const struct sysctlnode *cy_trng_node;
} sc_sysctl;
struct sun8i_crypto_opencrypto {
uint32_t co_driverid;
} sc_opencrypto;
};
struct sun8i_crypto_task {
struct sun8i_crypto_buf ct_descbuf;
struct sun8i_crypto_taskdesc *ct_desc;
struct sun8i_crypto_buf ct_ivbuf;
void *ct_iv;
struct sun8i_crypto_buf ct_ctrbuf;
void *ct_ctr;
bus_dmamap_t ct_descmap;
bus_dmamap_t ct_keymap;
bus_dmamap_t ct_ivmap; /* IV input */
bus_dmamap_t ct_ctrmap; /* updated IV output */
bus_dmamap_t ct_srcmap;
bus_dmamap_t ct_dstmap;
uint32_t ct_nbytes;
int ct_flags;
#define TASK_KEY __BIT(0)
#define TASK_IV __BIT(1)
#define TASK_CTR __BIT(2)
#define TASK_SRC __BIT(3)
#define TASK_BYTES __BIT(4) /* datalen is in bytes, not words */
void (*ct_callback)(struct sun8i_crypto_softc *,
struct sun8i_crypto_task *, void *, int);
void *ct_cookie;
};
#define SUN8I_CRYPTO_MAXDMASIZE PAGE_SIZE
#define SUN8I_CRYPTO_MAXDMASEGSIZE PAGE_SIZE
CTASSERT(SUN8I_CRYPTO_MAXDMASIZE <= SUN8I_CRYPTO_MAXDATALEN);
CTASSERT(SUN8I_CRYPTO_MAXDMASEGSIZE <= SUN8I_CRYPTO_MAXSEGLEN);
/*
* Forward declarations
*/
static int sun8i_crypto_match(device_t, cfdata_t, void *);
static void sun8i_crypto_attach(device_t, device_t, void *);
static int sun8i_crypto_task_ctor(void *, void *, int);
static void sun8i_crypto_task_dtor(void *, void *);
static struct sun8i_crypto_task *
sun8i_crypto_task_get(struct sun8i_crypto_softc *,
void (*)(struct sun8i_crypto_softc *,
struct sun8i_crypto_task *, void *, int),
void *, int);
static void sun8i_crypto_task_put(struct sun8i_crypto_softc *,
struct sun8i_crypto_task *);
static int sun8i_crypto_task_load(struct sun8i_crypto_softc *,
struct sun8i_crypto_task *, uint32_t,
uint32_t, uint32_t, uint32_t);
static int sun8i_crypto_task_scatter(struct sun8i_crypto_task *,
struct sun8i_crypto_adrlen *, bus_dmamap_t, uint32_t);
static int sun8i_crypto_task_load_trng(struct sun8i_crypto_softc *,
struct sun8i_crypto_task *, uint32_t);
static int sun8i_crypto_task_load_aesecb(struct sun8i_crypto_softc *,
struct sun8i_crypto_task *, uint32_t, uint32_t, uint32_t);
static int sun8i_crypto_submit(struct sun8i_crypto_softc *,
struct sun8i_crypto_task *);
static void sun8i_crypto_timeout(void *);
static int sun8i_crypto_intr(void *);
static void sun8i_crypto_schedule_worker(struct sun8i_crypto_softc *);
static void sun8i_crypto_worker(struct work *, void *);
static bool sun8i_crypto_poll(struct sun8i_crypto_softc *, uint32_t *,
uint32_t *);
static bool sun8i_crypto_done(struct sun8i_crypto_softc *, uint32_t,
uint32_t, unsigned);
static bool sun8i_crypto_chan_done(struct sun8i_crypto_softc *, unsigned,
int);
static int sun8i_crypto_allocbuf(struct sun8i_crypto_softc *, size_t,
struct sun8i_crypto_buf *, int);
static void sun8i_crypto_freebuf(struct sun8i_crypto_softc *, size_t,
struct sun8i_crypto_buf *);
static void sun8i_crypto_rng_attach(struct sun8i_crypto_softc *);
static void sun8i_crypto_rng_get(size_t, void *);
static void sun8i_crypto_rng_done(struct sun8i_crypto_softc *,
struct sun8i_crypto_task *, void *, int);
static bool sun8i_crypto_selftest(struct sun8i_crypto_softc *);
static void sun8i_crypto_selftest_done(struct sun8i_crypto_softc *,
struct sun8i_crypto_task *, void *, int);
static void sun8i_crypto_sysctl_attach(struct sun8i_crypto_softc *);
static int sun8i_crypto_sysctl_rng(SYSCTLFN_ARGS);
static void sun8i_crypto_sysctl_rng_done(struct sun8i_crypto_softc *,
struct sun8i_crypto_task *, void *, int);
static void sun8i_crypto_register(struct sun8i_crypto_softc *);
static void sun8i_crypto_register1(struct sun8i_crypto_softc *, uint32_t);
static int sun8i_crypto_newsession(void *, uint32_t *,
struct cryptoini *);
static void sun8i_crypto_freesession(void *, uint64_t);
static u_int sun8i_crypto_ivlen(const struct cryptodesc *);
static int sun8i_crypto_process(void *, struct cryptop *, int);
static void sun8i_crypto_callback(struct sun8i_crypto_softc *,
struct sun8i_crypto_task *, void *, int);
/*
* Probes
*/
SDT_PROBE_DEFINE2(sdt, sun8i_crypto, register, read,
"bus_size_t"/*reg*/,
"uint32_t"/*value*/);
SDT_PROBE_DEFINE2(sdt, sun8i_crypto, register, write,
"bus_size_t"/*reg*/,
"uint32_t"/*write*/);
SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, ctor__success,
"struct sun8i_crypto_task *"/*task*/);
SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, ctor__failure,
"int"/*error*/);
SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, dtor,
"struct sun8i_crypto_task *"/*task*/);
SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, get,
"struct sun8i_crypto_task *"/*task*/);
SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, put,
"struct sun8i_crypto_task *"/*task*/);
SDT_PROBE_DEFINE6(sdt, sun8i_crypto, task, load,
"struct sun8i_crypto_task *"/*task*/,
"uint32_t"/*tdqc*/,
"uint32_t"/*tdqs*/,
"uint32_t"/*tdqa*/,
"struct sun8i_crypto_taskdesc *"/*desc*/,
"int"/*error*/);
SDT_PROBE_DEFINE3(sdt, sun8i_crypto, task, misaligned,
"struct sun8i_crypto_task *"/*task*/,
"bus_addr_t"/*ds_addr*/,
"bus_size_t"/*ds_len*/);
SDT_PROBE_DEFINE2(sdt, sun8i_crypto, task, done,
"struct sun8i_crypto_task *"/*task*/,
"int"/*error*/);
SDT_PROBE_DEFINE3(sdt, sun8i_crypto, engine, submit__failure,
"struct sun8i_crypto_softc *"/*sc*/,
"struct sun8i_crypto_task *"/*task*/,
"int"/*error*/);
SDT_PROBE_DEFINE3(sdt, sun8i_crypto, engine, submit__success,
"struct sun8i_crypto_softc *"/*sc*/,
"struct sun8i_crypto_task *"/*task*/,
"unsigned"/*chan*/);
SDT_PROBE_DEFINE3(sdt, sun8i_crypto, engine, intr,
"struct sun8i_crypto_softc *"/*sc*/,
"uint32_t"/*isr*/,
"uint32_t"/*esr*/);
SDT_PROBE_DEFINE3(sdt, sun8i_crypto, engine, done,
"struct sun8i_crypto_softc *"/*sc*/,
"unsigned"/*chan*/,
"int"/*error*/);
SDT_PROBE_DEFINE3(sdt, sun8i_crypto, process, entry,
"struct sun8i_crypto_softc *"/*sc*/,
"struct cryptop *"/*crp*/,
"int"/*hint*/);
SDT_PROBE_DEFINE3(sdt, sun8i_crypto, process, busy,
"struct sun8i_crypto_softc *"/*sc*/,
"struct cryptop *"/*crp*/,
"int"/*hint*/);
SDT_PROBE_DEFINE4(sdt, sun8i_crypto, process, queued,
"struct sun8i_crypto_softc *"/*sc*/,
"struct cryptop *"/*crp*/,
"int"/*hint*/,
"struct sun8i_crypto_task *"/*task*/);
SDT_PROBE_DEFINE3(sdt, sun8i_crypto, process, done,
"struct sun8i_crypto_softc *"/*sc*/,
"struct cryptop *"/*crp*/,
"int"/*error*/);
/*
* Register access
*/
static uint32_t
sun8i_crypto_read(struct sun8i_crypto_softc *sc, bus_size_t reg)
{
uint32_t v = bus_space_read_4(sc->sc_bst, sc->sc_bsh, reg);
SDT_PROBE2(sdt, sun8i_crypto, register, read, reg, v);
return v;
}
static void
sun8i_crypto_write(struct sun8i_crypto_softc *sc, bus_size_t reg, uint32_t v)
{
SDT_PROBE2(sdt, sun8i_crypto, register, write, reg, v);
bus_space_write_4(sc->sc_bst, sc->sc_bsh, reg, v);
}
/*
* Autoconf goo
*/
CFATTACH_DECL_NEW(sun8i_crypto, sizeof(struct sun8i_crypto_softc),
sun8i_crypto_match, sun8i_crypto_attach, NULL, NULL);
static const struct device_compatible_entry compat_data[] = {
{ .compat = "allwinner,sun50i-a64-crypto",
.data = &sun50i_a64_crypto_config },
{ .compat = "allwinner,sun50i-h5-crypto",
.data = &sun50i_h5_crypto_config },
{ .compat = "allwinner,sun8i-h3-crypto",
.data = &sun8i_h3_crypto_config },
DEVICE_COMPAT_EOL
};
static int
sun8i_crypto_match(device_t parent, cfdata_t cf, void *aux)
{
const struct fdt_attach_args *const faa = aux;
return of_compatible_match(faa->faa_phandle, compat_data);
}
static void
sun8i_crypto_attach(device_t parent, device_t self, void *aux)
{
struct sun8i_crypto_softc *const sc = device_private(self);
const struct fdt_attach_args *const faa = aux;
bus_addr_t addr;
bus_size_t size;
const int phandle = faa->faa_phandle;
char intrstr[128];
struct clk *clk;
struct fdtbus_reset *rst;
u_int mod_rate;
sc->sc_dev = self;
sc->sc_dmat = faa->faa_dmat;
sc->sc_bst = faa->faa_bst;
sc->sc_taskpool = pool_cache_init(sizeof(struct sun8i_crypto_task),
0, 0, 0, "sun8icry", NULL, IPL_SOFTSERIAL,
&sun8i_crypto_task_ctor, &sun8i_crypto_task_dtor, sc);
sc->sc_cfg = of_compatible_lookup(phandle, compat_data)->data;
mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_SOFTSERIAL);
mutex_init(&sc->sc_intr_lock, MUTEX_DEFAULT, IPL_VM);
callout_init(&sc->sc_timeout, CALLOUT_MPSAFE);
callout_setfunc(&sc->sc_timeout, &sun8i_crypto_timeout, sc);
if (workqueue_create(&sc->sc_wq, device_xname(self),
&sun8i_crypto_worker, sc, PRI_NONE, IPL_VM, WQ_MPSAFE) != 0) {
aprint_error(": couldn't create workqueue\n");
return;
}
/*
* Prime the pool with enough tasks that each channel can be
* busy with a task as we prepare another task for when it's
* done.
*/
pool_cache_prime(sc->sc_taskpool, 2*SUN8I_CRYPTO_NCHAN);
/* Get and map device registers. */
if (fdtbus_get_reg(phandle, 0, &addr, &size) != 0) {
aprint_error(": couldn't get registers\n");
return;
}
if (bus_space_map(sc->sc_bst, addr, size, 0, &sc->sc_bsh) != 0) {
aprint_error(": couldn't map registers\n");
return;
}
/* Get an interrupt handle. */
if (!fdtbus_intr_str(phandle, 0, intrstr, sizeof(intrstr))) {
aprint_error(": failed to decode interrupt\n");
return;
}
/* Enable the bus clock. */
if (fdtbus_clock_enable(phandle, "bus", true) != 0) {
aprint_error(": couldn't enable bus clock\n");
return;
}
/* Get the module clock and set it. */
mod_rate = sc->sc_cfg->mod_rate;
if ((clk = fdtbus_clock_get(phandle, "mod")) != NULL) {
if (clk_enable(clk) != 0) {
aprint_error(": couldn't enable CE clock\n");
return;
}
if (clk_set_rate(clk, mod_rate) != 0) {
aprint_error(": couldn't set CE clock to %d MHz\n",
mod_rate / (1000 * 1000));
return;
}
}
/* Get a reset handle if we need and try to deassert it. */
if ((rst = fdtbus_reset_get_index(phandle, 0)) != NULL) {
if (fdtbus_reset_deassert(rst) != 0) {
aprint_error(": couldn't de-assert reset\n");
return;
}
}
aprint_naive("\n");
aprint_normal(": Crypto Engine\n");
aprint_debug_dev(self, ": clock freq %d\n", clk_get_rate(clk));
/*
* Disable and clear interrupts. Start in polling mode for
* synchronous self-tests and the first RNG draw.
*/
sun8i_crypto_write(sc, SUN8I_CRYPTO_ICR, 0);
sun8i_crypto_write(sc, SUN8I_CRYPTO_ISR, 0);
sc->sc_polling = true;
/* Establish an interrupt handler. */
sc->sc_ih = fdtbus_intr_establish_xname(phandle, 0, IPL_VM,
FDT_INTR_MPSAFE, &sun8i_crypto_intr, sc, device_xname(self));
if (sc->sc_ih == NULL) {
aprint_error_dev(self, "failed to establish interrupt on %s\n",
intrstr);
return;
}
aprint_normal_dev(self, "interrupting on %s\n", intrstr);
/* Perform self-tests. If they fail, stop here. */
if (!sun8i_crypto_selftest(sc))
return;
/*
* Set up the RNG. This will try to synchronously draw the
* first sample by polling, so do this before we establish
* the interrupt handler.
*/
sun8i_crypto_rng_attach(sc);
/*
* Self-test has passed and first RNG draw has finished. Use
* interrupts, not polling, for all subsequent tasks. Set this
* atomically in case the interrupt handler has fired -- can't
* be from us because we've kept ICR set to 0 to mask all
* interrupts, but in case the interrupt vector is shared.
*/
atomic_store_relaxed(&sc->sc_polling, false);
/* Attach the sysctl. */
sun8i_crypto_sysctl_attach(sc);
/* Register opencrypto handlers. */
sun8i_crypto_register(sc);
}
static int
sun8i_crypto_task_ctor(void *cookie, void *vtask, int pflags)
{
struct sun8i_crypto_softc *sc = cookie;
struct sun8i_crypto_task *task = vtask;
int dmaflags = (pflags & PR_WAITOK) ? BUS_DMA_WAITOK : BUS_DMA_NOWAIT;
int error;
/* Create a DMA buffer for the task descriptor. */
error = sun8i_crypto_allocbuf(sc, sizeof(*task->ct_desc),
&task->ct_descbuf, dmaflags);
if (error)
goto fail0;
task->ct_desc = task->ct_descbuf.cb_kva;
/* Create DMA buffers for the IV and CTR. */
error = sun8i_crypto_allocbuf(sc, SUN8I_CRYPTO_MAXIVBYTES,
&task->ct_ivbuf, dmaflags);
if (error)
goto fail1;
task->ct_iv = task->ct_ivbuf.cb_kva;
error = sun8i_crypto_allocbuf(sc, SUN8I_CRYPTO_MAXCTRBYTES,
&task->ct_ctrbuf, dmaflags);
if (error)
goto fail2;
task->ct_ctr = task->ct_ctrbuf.cb_kva;
/* Create a DMA map for the task descriptor and preload it. */
error = bus_dmamap_create(sc->sc_dmat, sizeof(*task->ct_desc), 1,
sizeof(*task->ct_desc), 0, dmaflags, &task->ct_descmap);
if (error)
goto fail3;
error = bus_dmamap_load(sc->sc_dmat, task->ct_descmap, task->ct_desc,
sizeof(*task->ct_desc), NULL, BUS_DMA_WAITOK);
if (error)
goto fail4;
/* Create DMA maps for the key, IV, and CTR. */
error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXKEYBYTES, 1,
SUN8I_CRYPTO_MAXKEYBYTES, 0, dmaflags, &task->ct_keymap);
if (error)
goto fail5;
error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXIVBYTES, 1,
SUN8I_CRYPTO_MAXIVBYTES, 0, dmaflags, &task->ct_ivmap);
if (error)
goto fail6;
error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXCTRBYTES, 1,
SUN8I_CRYPTO_MAXCTRBYTES, 0, dmaflags, &task->ct_ctrmap);
if (error)
goto fail7;
/* Create DMA maps for the src and dst scatter/gather vectors. */
error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXDMASIZE,
SUN8I_CRYPTO_MAXSEGS, SUN8I_CRYPTO_MAXDMASEGSIZE, 0, dmaflags,
&task->ct_srcmap);
if (error)
goto fail8;
error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXDMASIZE,
SUN8I_CRYPTO_MAXSEGS, SUN8I_CRYPTO_MAXDMASEGSIZE, 0, dmaflags,
&task->ct_dstmap);
if (error)
goto fail9;
/* Success! */
SDT_PROBE1(sdt, sun8i_crypto, task, ctor__success, task);
return 0;
fail10: __unused
bus_dmamap_destroy(sc->sc_dmat, task->ct_dstmap);
fail9: bus_dmamap_destroy(sc->sc_dmat, task->ct_srcmap);
fail8: bus_dmamap_destroy(sc->sc_dmat, task->ct_ctrmap);
fail7: bus_dmamap_destroy(sc->sc_dmat, task->ct_ivmap);
fail6: bus_dmamap_destroy(sc->sc_dmat, task->ct_keymap);
fail5: bus_dmamap_unload(sc->sc_dmat, task->ct_descmap);
fail4: bus_dmamap_destroy(sc->sc_dmat, task->ct_descmap);
fail3: sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_MAXIVBYTES, &task->ct_ivbuf);
fail2: sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_MAXCTRBYTES, &task->ct_ctrbuf);
fail1: sun8i_crypto_freebuf(sc, sizeof(*task->ct_desc), &task->ct_descbuf);
fail0: SDT_PROBE1(sdt, sun8i_crypto, task, ctor__failure, error);
return error;
}
static void
sun8i_crypto_task_dtor(void *cookie, void *vtask)
{
struct sun8i_crypto_softc *sc = cookie;
struct sun8i_crypto_task *task = vtask;
SDT_PROBE1(sdt, sun8i_crypto, task, dtor, task);
/* XXX Zero the bounce buffers if there are any. */
bus_dmamap_destroy(sc->sc_dmat, task->ct_dstmap);
bus_dmamap_destroy(sc->sc_dmat, task->ct_srcmap);
bus_dmamap_destroy(sc->sc_dmat, task->ct_ctrmap);
bus_dmamap_destroy(sc->sc_dmat, task->ct_ivmap);
bus_dmamap_destroy(sc->sc_dmat, task->ct_keymap);
bus_dmamap_unload(sc->sc_dmat, task->ct_descmap);
bus_dmamap_destroy(sc->sc_dmat, task->ct_descmap);
sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_MAXIVBYTES, &task->ct_ivbuf);
sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_MAXCTRBYTES, &task->ct_ctrbuf);
sun8i_crypto_freebuf(sc, sizeof(*task->ct_desc), &task->ct_descbuf);
}
/*
* sun8i_crypto_task_get(sc, callback, cookie, pflags)
*
* Allocate a task that will call callback(sc, task, cookie,
* error) when done. pflags is PR_WAITOK or PR_NOWAIT; if
* PR_NOWAIT, may fail and return NULL. No further allocation is
* needed to submit the task if this succeeds (although task
* submission may still fail if all channels are busy).
*/
static struct sun8i_crypto_task *
sun8i_crypto_task_get(struct sun8i_crypto_softc *sc,
void (*callback)(struct sun8i_crypto_softc *, struct sun8i_crypto_task *,
void *, int),
void *cookie, int pflags)
{
struct sun8i_crypto_task *task;
/* Allocate a task, or fail if we can't. */
task = pool_cache_get(sc->sc_taskpool, pflags);
if (task == NULL)
goto out;
/* Set up flags and the callback. */
task->ct_flags = 0;
task->ct_callback = callback;
task->ct_cookie = cookie;
out: SDT_PROBE1(sdt, sun8i_crypto, task, get, task);
return task;
}
/*
* sun8i_crypto_task_invalid(sc, task, cookie, error)
*
* Callback for a task not currently in use, to detect errors.
*/
static void
sun8i_crypto_task_invalid(struct sun8i_crypto_softc *sc,
struct sun8i_crypto_task *task, void *cookie, int error)
{
void (*callback)(struct sun8i_crypto_softc *,
struct sun8i_crypto_task *, void *, int) = cookie;
panic("task for callback %p used after free", callback);
}
/*
* sun8i_crypto_task_put(sc, task)
*
* Free a task obtained with sun8i_crypto_task_get.
*/
static void
sun8i_crypto_task_put(struct sun8i_crypto_softc *sc,
struct sun8i_crypto_task *task)
{
SDT_PROBE1(sdt, sun8i_crypto, task, put, task);
task->ct_cookie = task->ct_callback;
task->ct_callback = &sun8i_crypto_task_invalid;
pool_cache_put(sc->sc_taskpool, task);
}
/*
* sun8i_crypto_task_load(sc, task, nbytes, tdqc, tdqs, tdqa)
*
* Set up the task descriptor after the relevant DMA maps have
* been loaded for a transfer of nbytes. bus_dmamap_sync matches
* sun8i_crypto_chan_done. May fail if input is inadequately
* aligned.
*
* XXX Teach this to support task chains.
*/
static int
sun8i_crypto_task_load(struct sun8i_crypto_softc *sc,
struct sun8i_crypto_task *task, uint32_t nbytes,
uint32_t tdqc, uint32_t tdqs, uint32_t tdqa)
{
struct sun8i_crypto_taskdesc *desc = task->ct_desc;
int error;
KASSERT(tdqs == 0 || tdqa == 0);
KASSERT(nbytes % 4 == 0);
memset(desc, 0, sizeof(*desc));
/* Always enable interrupt for the task. */
tdqc |= SUN8I_CRYPTO_TDQC_INTR_EN;
desc->td_tdqc = htole32(tdqc);
desc->td_tdqs = htole32(tdqs);
desc->td_tdqa = htole32(tdqa);
if (task->ct_flags & TASK_KEY) {
bus_dmamap_t keymap = task->ct_keymap;
KASSERT(keymap->dm_nsegs == 1);
desc->td_keydesc = htole32(keymap->dm_segs[0].ds_addr);
bus_dmamap_sync(sc->sc_dmat, keymap, 0,
keymap->dm_segs[0].ds_len, BUS_DMASYNC_PREWRITE);
}
if (task->ct_flags & TASK_IV) {
bus_dmamap_t ivmap = task->ct_ivmap;
KASSERT(ivmap->dm_nsegs == 1);
desc->td_ivdesc = htole32(ivmap->dm_segs[0].ds_addr);
bus_dmamap_sync(sc->sc_dmat, ivmap, 0,
ivmap->dm_segs[0].ds_len, BUS_DMASYNC_PREWRITE);
}
if (task->ct_flags & TASK_CTR) {
bus_dmamap_t ctrmap = task->ct_ctrmap;
KASSERT(ctrmap->dm_nsegs == 1);
desc->td_ctrdesc = htole32(ctrmap->dm_segs[0].ds_addr);
bus_dmamap_sync(sc->sc_dmat, ctrmap, 0,
ctrmap->dm_segs[0].ds_len, BUS_DMASYNC_PREREAD);
}
if (task->ct_flags & TASK_BYTES)
desc->td_datalen = htole32(nbytes);
else
desc->td_datalen = htole32(nbytes/4);
if (task->ct_flags & TASK_SRC) {
bus_dmamap_t srcmap = task->ct_srcmap;
KASSERT(srcmap->dm_mapsize == task->ct_dstmap->dm_mapsize);
error = sun8i_crypto_task_scatter(task, desc->td_src, srcmap,
nbytes);
if (error)
return error;
bus_dmamap_sync(sc->sc_dmat, srcmap, 0, nbytes,
BUS_DMASYNC_PREWRITE);
}
error = sun8i_crypto_task_scatter(task, desc->td_dst, task->ct_dstmap,
nbytes);
if (error)
goto out;
bus_dmamap_sync(sc->sc_dmat, task->ct_dstmap, 0, nbytes,
BUS_DMASYNC_PREREAD);
task->ct_nbytes = nbytes;
/* Success! */
error = 0;
out: SDT_PROBE6(sdt, sun8i_crypto, task, load,
task, tdqc, tdqs, tdqa, desc, error);
return error;
}
/*
* sun8i_crypto_task_scatter(task, adrlen, map, nbytes)
*
* Set up a task's scatter/gather vector -- src or dst -- with the
* given DMA map for a transfer of nbytes. May fail if input is
* inadequately aligned.
*/
static int
sun8i_crypto_task_scatter(struct sun8i_crypto_task *task,
struct sun8i_crypto_adrlen *adrlen, bus_dmamap_t map,
uint32_t nbytes __diagused)
{
uint32_t total __diagused = 0;
unsigned i;
/*
* Verify that the alignment is correct and initialize the
* scatter/gather vector.
*/
KASSERT(map->dm_nsegs <= SUN8I_CRYPTO_MAXSEGS);
for (i = 0; i < map->dm_nsegs; i++) {
if ((map->dm_segs[i].ds_addr % 4) |
(map->dm_segs[i].ds_len % 4)) {
SDT_PROBE3(sdt, sun8i_crypto, task, misaligned,
task,
map->dm_segs[i].ds_addr,
map->dm_segs[i].ds_len);
return EINVAL;
}
KASSERT(map->dm_segs[i].ds_addr <= UINT32_MAX);
KASSERT(map->dm_segs[i].ds_len <= UINT32_MAX - total);
adrlen[i].adr = htole32(map->dm_segs[i].ds_addr);
adrlen[i].len = htole32(map->dm_segs[i].ds_len/4);
total += map->dm_segs[i].ds_len;
}
/* Set the remainder to zero. */
for (; i < SUN8I_CRYPTO_MAXSEGS; i++) {
adrlen[i].adr = 0;
adrlen[i].len = 0;
}
/* Verify the total size matches the transfer length. */
KASSERT(total == nbytes);
/* Success! */
return 0;
}
/*
* sun8i_crypto_task_load_trng(task, nbytes)
*
* Set up the task descriptor for a transfer of nbytes from the
* TRNG.
*/
static int
sun8i_crypto_task_load_trng(struct sun8i_crypto_softc *sc,
struct sun8i_crypto_task *task, uint32_t nbytes)
{
uint32_t tdqc = 0;
/* Caller must provide dst only. */
KASSERT((task->ct_flags & TASK_KEY) == 0);
KASSERT((task->ct_flags & TASK_IV) == 0);
KASSERT((task->ct_flags & TASK_CTR) == 0);
KASSERT((task->ct_flags & TASK_SRC) == 0);
/* Set up the task descriptor queue control words. */
tdqc |= __SHIFTIN(SUN8I_CRYPTO_TDQC_METHOD_TRNG,
SUN8I_CRYPTO_TDQC_METHOD);
/* Fill in the descriptor. */
return sun8i_crypto_task_load(sc, task, nbytes, tdqc, 0, 0);
}
static int
sun8i_crypto_task_load_aesecb(struct sun8i_crypto_softc *sc,
struct sun8i_crypto_task *task,
uint32_t nbytes, uint32_t keysize, uint32_t dir)
{
uint32_t tdqc = 0, tdqs = 0;
/* Caller must provide key, src, and dst only. */
KASSERT(task->ct_flags & TASK_KEY);
KASSERT((task->ct_flags & TASK_IV) == 0);
KASSERT((task->ct_flags & TASK_CTR) == 0);
KASSERT(task->ct_flags & TASK_SRC);
/* Set up the task descriptor queue control word. */
tdqc |= __SHIFTIN(SUN8I_CRYPTO_TDQC_METHOD_AES,
SUN8I_CRYPTO_TDQC_METHOD);
tdqc |= __SHIFTIN(dir, SUN8I_CRYPTO_TDQC_OP_DIR);
#ifdef DIAGNOSTIC
switch (keysize) {
case SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128:
KASSERT(task->ct_keymap->dm_segs[0].ds_len == 16);
break;
case SUN8I_CRYPTO_TDQS_AES_KEYSIZE_192:
KASSERT(task->ct_keymap->dm_segs[0].ds_len == 24);
break;
case SUN8I_CRYPTO_TDQS_AES_KEYSIZE_256:
KASSERT(task->ct_keymap->dm_segs[0].ds_len == 32);
break;
}
#endif
/* Set up the symmetric control word. */
tdqs |= __SHIFTIN(SUN8I_CRYPTO_TDQS_SKEY_SELECT_SS_KEYx,
SUN8I_CRYPTO_TDQS_SKEY_SELECT);
tdqs |= __SHIFTIN(SUN8I_CRYPTO_TDQS_OP_MODE_ECB,
SUN8I_CRYPTO_TDQS_OP_MODE);
tdqs |= __SHIFTIN(keysize, SUN8I_CRYPTO_TDQS_AES_KEYSIZE);
/* Fill in the descriptor. */
return sun8i_crypto_task_load(sc, task, nbytes, tdqc, tdqs, 0);
}
/*
* sun8i_crypto_submit(sc, task)
*
* Submit a task to the crypto engine after it has been loaded
* with sun8i_crypto_task_load. On success, guarantees to
* eventually call the task's callback.
*/
static int
sun8i_crypto_submit(struct sun8i_crypto_softc *sc,
struct sun8i_crypto_task *task)
{
unsigned i, retries = 0;
uint32_t icr;
int error = 0;
/* One at a time at the device registers, please. */
mutex_enter(&sc->sc_lock);
/* Find a channel. */
for (i = 0; i < SUN8I_CRYPTO_NCHAN; i++) {
if (sc->sc_chan[i].cc_task == NULL)
break;
}
if (i == SUN8I_CRYPTO_NCHAN) {
device_printf(sc->sc_dev, "no free channels\n");
error = ERESTART;
goto out;
}
/*
* Set the channel id. Caller is responsible for setting up
* all other parts of the descriptor.
*/
task->ct_desc->td_cid = htole32(i);
/*
* Prepare to send the descriptor to the device by DMA.
* Matches POSTWRITE in sun8i_crypto_chan_done.
*/
bus_dmamap_sync(sc->sc_dmat, task->ct_descmap, 0,
sizeof(*task->ct_desc), BUS_DMASYNC_PREWRITE);
/* Confirm we're ready to go. */
if (sun8i_crypto_read(sc, SUN8I_CRYPTO_TLR) & SUN8I_CRYPTO_TLR_LOAD) {
device_printf(sc->sc_dev, "TLR not clear\n");
error = EIO;
goto out;
}
/*
* Enable interrupts for this channel, unless we're still
* polling.
*/
if (!sc->sc_polling) {
icr = sun8i_crypto_read(sc, SUN8I_CRYPTO_ICR);
icr |= __SHIFTIN(SUN8I_CRYPTO_ICR_INTR_EN_CHAN(i),
SUN8I_CRYPTO_ICR_INTR_EN);
sun8i_crypto_write(sc, SUN8I_CRYPTO_ICR, icr);
}
/* Set the task descriptor queue address. */
sun8i_crypto_write(sc, SUN8I_CRYPTO_TDQ,
task->ct_descmap->dm_segs[0].ds_addr);
/* Notify the engine to load it, and wait for acknowledgement. */
sun8i_crypto_write(sc, SUN8I_CRYPTO_TLR, SUN8I_CRYPTO_TLR_LOAD);
while (sun8i_crypto_read(sc, SUN8I_CRYPTO_TLR) & SUN8I_CRYPTO_TLR_LOAD)
{
/*
* XXX Timeout pulled from arse. Is it even important
* to wait here?
*/
if (++retries == 1000) {
device_printf(sc->sc_dev, "TLR didn't clear: %08x\n",
sun8i_crypto_read(sc, SUN8I_CRYPTO_TLR));
/*
* Hope it clears eventually; if not, we'll
* time out.
*/
break;
}
DELAY(1);
}
/*
* Loaded up and ready to go. Start a timer ticking if it's
* not already and we're not polling.
*/
sc->sc_chan[i].cc_task = task;
sc->sc_chan[i].cc_starttime = getticks();
if (!sc->sc_polling && !callout_pending(&sc->sc_timeout))
callout_schedule(&sc->sc_timeout, SUN8I_CRYPTO_TIMEOUT);
out: /* Done! */
if (error)
SDT_PROBE3(sdt, sun8i_crypto, engine, submit__failure,
sc, task, error);
else
SDT_PROBE3(sdt, sun8i_crypto, engine, submit__success,
sc, task, i);
mutex_exit(&sc->sc_lock);
return error;
}
/*
* sun8i_crypto_timeout(cookie)
*
* Timeout handler. Schedules work in a thread to cancel all
* pending tasks that were started long enough ago we're bored of
* waiting for them.
*/
static void
sun8i_crypto_timeout(void *cookie)
{
struct sun8i_crypto_softc *sc = cookie;
mutex_enter(&sc->sc_intr_lock);
sun8i_crypto_schedule_worker(sc);
mutex_exit(&sc->sc_intr_lock);
}
/*
* sun8i_crypto_intr(cookie)
*
* Device interrupt handler. Find what channels have completed,
* whether with success or with failure, and schedule work in
* thread context to invoke the appropriate callbacks.
*/
static int
sun8i_crypto_intr(void *cookie)
{
struct sun8i_crypto_softc *sc = cookie;
uint32_t done, esr;
if (atomic_load_relaxed(&sc->sc_polling) ||
!sun8i_crypto_poll(sc, &done, &esr))
return 0; /* not ours */
mutex_enter(&sc->sc_intr_lock);
sun8i_crypto_schedule_worker(sc);
sc->sc_done |= done;
sc->sc_esr |= esr;
mutex_exit(&sc->sc_intr_lock);
return 1;
}
/*
* sun8i_crypto_schedule_worker(sc)
*
* Ensure that crypto engine thread context work to invoke task
* callbacks will run promptly. Idempotent.
*/
static void
sun8i_crypto_schedule_worker(struct sun8i_crypto_softc *sc)
{
KASSERT(mutex_owned(&sc->sc_intr_lock));
/* Start the worker if necessary. */
if (!sc->sc_work_pending) {
workqueue_enqueue(sc->sc_wq, &sc->sc_work, NULL);
sc->sc_work_pending = true;
}
}
/*
* sun8i_crypto_worker(wk, cookie)
*
* Thread-context worker: Invoke all task callbacks for which the
* device has notified us of completion or for which we gave up
* waiting.
*/
static void
sun8i_crypto_worker(struct work *wk, void *cookie)
{
struct sun8i_crypto_softc *sc = cookie;
uint32_t done, esr;
/*
* Under the interrupt lock, acknowledge our work and claim the
* done mask and error status.
*/
mutex_enter(&sc->sc_intr_lock);
KASSERT(sc->sc_work_pending);
sc->sc_work_pending = false;
done = sc->sc_done;
esr = sc->sc_esr;
sc->sc_done = 0;
sc->sc_esr = 0;
mutex_exit(&sc->sc_intr_lock);
/*
* If we cleared any channels, it is time to allow opencrypto
* to issue new operations. Asymmetric operations (which we
* don't support, at the moment, but we could) and symmetric
* operations (which we do) use the same task channels, so we
* unblock both kinds.
*/
if (sun8i_crypto_done(sc, done, esr, getticks())) {
crypto_unblock(sc->sc_opencrypto.co_driverid,
CRYPTO_SYMQ|CRYPTO_ASYMQ);
}
}
/*
* sun8i_crypto_poll(sc, &done, &esr)
*
* Poll for completion. Sets done and esr to the mask of done
* channels and error channels. Returns true if anything was
* done or failed.
*/
static bool
sun8i_crypto_poll(struct sun8i_crypto_softc *sc,
uint32_t *donep, uint32_t *esrp)
{
uint32_t isr, esr;
/*
* Get and acknowledge the interrupts and error status.
*
* XXX Data sheet says the error status register is read-only,
* but then advises writing 1 to bit x1xx (keysram access error
* for AES, SUN8I_CRYPTO_ESR_KEYSRAMERR) to clear it. What do?
*/
isr = sun8i_crypto_read(sc, SUN8I_CRYPTO_ISR);
esr = sun8i_crypto_read(sc, SUN8I_CRYPTO_ESR);
sun8i_crypto_write(sc, SUN8I_CRYPTO_ISR, isr);
sun8i_crypto_write(sc, SUN8I_CRYPTO_ESR, esr);
SDT_PROBE3(sdt, sun8i_crypto, engine, intr, sc, isr, esr);
*donep = __SHIFTOUT(isr, SUN8I_CRYPTO_ISR_DONE);
*esrp = esr;
return *donep || *esrp;
}
/*
* sun8i_crypto_done(sc, done, esr, now)
*
* Invoke all task callbacks for the channels in done or esr, or
* for which we gave up waiting, according to the time `now'.
* Returns true if any channels completed or timed out.
*/
static bool
sun8i_crypto_done(struct sun8i_crypto_softc *sc, uint32_t done, uint32_t esr,
unsigned now)
{
uint32_t esr_chan;
unsigned i;
bool anydone = false;
bool schedtimeout = false;
int error;
/* Under the lock, process the channels. */
mutex_enter(&sc->sc_lock);
for (i = 0; i < SUN8I_CRYPTO_NCHAN; i++) {
/* Check whether the channel is done. */
if (!ISSET(done, SUN8I_CRYPTO_ISR_DONE_CHAN(i))) {
/* Nope. Do we have a task to time out? */
if (sc->sc_chan[i].cc_task != NULL) {
if (now - sc->sc_chan[i].cc_starttime >=
SUN8I_CRYPTO_TIMEOUT) {
anydone |= sun8i_crypto_chan_done(sc,
i, ETIMEDOUT);
} else {
schedtimeout = true;
}
}
continue;
}
/* Channel is done. Interpret the error if any. */
esr_chan = __SHIFTOUT(esr, SUN8I_CRYPTO_ESR_CHAN(i));
if (esr_chan & SUN8I_CRYPTO_ESR_CHAN_ALGNOTSUP) {
device_printf(sc->sc_dev, "channel %u:"
" alg not supported\n", i);
error = ENODEV;
} else if (esr_chan & SUN8I_CRYPTO_ESR_CHAN_DATALENERR) {
device_printf(sc->sc_dev, "channel %u:"
" data length error\n", i);
error = EIO; /* XXX */
} else if (esr_chan & SUN8I_CRYPTO_ESR_CHAN_KEYSRAMERR) {
device_printf(sc->sc_dev, "channel %u:"
" key sram error\n", i);
error = EIO; /* XXX */
} else if (esr_chan != 0) {
error = EIO; /* generic I/O error */
} else {
error = 0;
}
/*
* Notify the task of completion. May release the lock
* to invoke a callback.
*/
anydone |= sun8i_crypto_chan_done(sc, i, error);
}
mutex_exit(&sc->sc_lock);
/*
* If there are tasks still pending, make sure there's a
* timeout scheduled for them. If the callout is already
* pending, it will take another pass through here to time some
* things out and schedule a new timeout.
*/
if (schedtimeout && !callout_pending(&sc->sc_timeout))
callout_schedule(&sc->sc_timeout, SUN8I_CRYPTO_TIMEOUT);
return anydone;
}
/*
* sun8i_crypto_chan_done(sc, i, error)
*
* Notify the callback for the task on channel i, if there is one,
* of the specified error, or 0 for success.
*/
static bool
sun8i_crypto_chan_done(struct sun8i_crypto_softc *sc, unsigned i, int error)
{
struct sun8i_crypto_task *task;
uint32_t nbytes;
uint32_t icr;
KASSERT(mutex_owned(&sc->sc_lock));
SDT_PROBE3(sdt, sun8i_crypto, engine, done, sc, i, error);
/* Claim the task if there is one; bail if not. */
if ((task = sc->sc_chan[i].cc_task) == NULL) {
device_printf(sc->sc_dev, "channel %u: no task but error=%d\n",
i, error);
/* We did not clear a channel. */
return false;
}
sc->sc_chan[i].cc_task = NULL;
/* Disable interrupts on this channel. */
icr = sun8i_crypto_read(sc, SUN8I_CRYPTO_ICR);
icr &= ~__SHIFTIN(SUN8I_CRYPTO_ICR_INTR_EN_CHAN(i),
SUN8I_CRYPTO_ICR_INTR_EN);
sun8i_crypto_write(sc, SUN8I_CRYPTO_ICR, icr);
/*
* Finished sending the descriptor to the device by DMA.
* Matches PREWRITE in sun8i_crypto_task_submit.
*/
bus_dmamap_sync(sc->sc_dmat, task->ct_descmap, 0,
sizeof(*task->ct_desc), BUS_DMASYNC_POSTWRITE);
/*
* Finished with all the other bits of DMA too. Matches
* sun8i_crypto_task_load.
*/
nbytes = task->ct_nbytes;
bus_dmamap_sync(sc->sc_dmat, task->ct_dstmap, 0, nbytes,
BUS_DMASYNC_POSTREAD);
if (task->ct_flags & TASK_SRC)
bus_dmamap_sync(sc->sc_dmat, task->ct_srcmap, 0, nbytes,
BUS_DMASYNC_POSTWRITE);
if (task->ct_flags & TASK_CTR)
bus_dmamap_sync(sc->sc_dmat, task->ct_ctrmap, 0,
task->ct_ctrmap->dm_segs[0].ds_len, BUS_DMASYNC_POSTREAD);
if (task->ct_flags & TASK_IV)
bus_dmamap_sync(sc->sc_dmat, task->ct_ivmap, 0,
task->ct_ivmap->dm_segs[0].ds_len, BUS_DMASYNC_POSTWRITE);
if (task->ct_flags & TASK_KEY)
/* XXX Can we zero the bounce buffer if there is one? */
bus_dmamap_sync(sc->sc_dmat, task->ct_keymap, 0,
task->ct_keymap->dm_segs[0].ds_len, BUS_DMASYNC_POSTWRITE);
/* Temporarily release the lock to invoke the callback. */
mutex_exit(&sc->sc_lock);
SDT_PROBE2(sdt, sun8i_crypto, task, done, task, error);
(*task->ct_callback)(sc, task, task->ct_cookie, error);
mutex_enter(&sc->sc_lock);
/* We cleared a channel. */
return true;
}
/*
* sun8i_crypto_allocbuf(sc, size, buf, dmaflags)
*
* Allocate a single-segment DMA-safe buffer and map it into KVA.
* May fail if dmaflags is BUS_DMA_NOWAIT.
*/
static int
sun8i_crypto_allocbuf(struct sun8i_crypto_softc *sc, size_t size,
struct sun8i_crypto_buf *buf, int dmaflags)
{
int error;
/* Allocate a DMA-safe buffer. */
error = bus_dmamem_alloc(sc->sc_dmat, size, sizeof(uint32_t), 0,
buf->cb_seg, __arraycount(buf->cb_seg), &buf->cb_nsegs, dmaflags);
if (error)
goto fail0;
/* Map the buffer into kernel virtual address space. */
error = bus_dmamem_map(sc->sc_dmat, buf->cb_seg, buf->cb_nsegs,
size, &buf->cb_kva, dmaflags);
if (error)
goto fail1;
/* Success! */
return 0;
fail2: __unused
bus_dmamem_unmap(sc->sc_dmat, buf->cb_kva, size);
fail1: bus_dmamem_free(sc->sc_dmat, buf->cb_seg, buf->cb_nsegs);
fail0: return error;
}
/*
* sun8i_crypto_freebuf(sc, buf)
*
* Unmap buf and free it.
*/
static void
sun8i_crypto_freebuf(struct sun8i_crypto_softc *sc, size_t size,
struct sun8i_crypto_buf *buf)
{
bus_dmamem_unmap(sc->sc_dmat, buf->cb_kva, size);
bus_dmamem_free(sc->sc_dmat, buf->cb_seg, buf->cb_nsegs);
}
/*
* sun8i_crypto_rng_attach(sc)
*
* Attach an rndsource for the crypto engine's TRNG.
*/
static void
sun8i_crypto_rng_attach(struct sun8i_crypto_softc *sc)
{
device_t self = sc->sc_dev;
struct sun8i_crypto_rng *rng = &sc->sc_rng;
struct sun8i_crypto_task *task;
unsigned timo = hztoms(SUN8I_CRYPTO_TIMEOUT);
uint32_t done, esr;
int error;
/* Preallocate a buffer to reuse. */
error = sun8i_crypto_allocbuf(sc, SUN8I_CRYPTO_RNGBYTES, &rng->cr_buf,
BUS_DMA_WAITOK);
if (error) {
aprint_error_dev(self, "failed to allocate RNG buffer: %d\n",
error);
goto fail0;
}
/* Create a task to reuse. */
task = rng->cr_task = sun8i_crypto_task_get(sc, sun8i_crypto_rng_done,
rng, PR_WAITOK);
if (rng->cr_task == NULL) {
aprint_error_dev(self, "failed to allocate RNG task\n");
error = ENOMEM;
goto fail1;
}
/* Preload the destination map. */
error = bus_dmamap_load(sc->sc_dmat, task->ct_dstmap,
rng->cr_buf.cb_kva, SUN8I_CRYPTO_RNGBYTES, NULL, BUS_DMA_NOWAIT);
if (error) {
aprint_error_dev(self, "failed to load RNG buffer: %d\n",
error);
goto fail2;
}
/*
* Attach the rndsource. This will trigger an initial call to
* it since we have RND_FLAG_HASCB.
*/
rndsource_setcb(&rng->cr_rndsource, sun8i_crypto_rng_get, sc);
rnd_attach_source(&rng->cr_rndsource, device_xname(self),
RND_TYPE_RNG,
RND_FLAG_COLLECT_VALUE|RND_FLAG_ESTIMATE_VALUE|RND_FLAG_HASCB);
/*
* Poll for the first call to the RNG to complete. If not done
* after the timeout, force a timeout.
*/
for (; rng->cr_pending && timo --> 0; DELAY(1000)) {
if (sun8i_crypto_poll(sc, &done, &esr))
(void)sun8i_crypto_done(sc, done, esr, getticks());
}
if (rng->cr_pending) {
(void)sun8i_crypto_done(sc, 0, 0,
(unsigned)getticks() + SUN8I_CRYPTO_TIMEOUT);
KASSERT(!rng->cr_pending);
}
return;
fail3: __unused
bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap);
fail2: sun8i_crypto_task_put(sc, task);
fail1: sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_RNGBYTES, &rng->cr_buf);
fail0: return;
}
/*
* sun8i_crypto_rng_get(nbytes, cookie)
*
* On-demand rndsource callback: try to gather nbytes of entropy
* and enter them into the pool ASAP.
*/
static void
sun8i_crypto_rng_get(size_t nbytes, void *cookie)
{
struct sun8i_crypto_softc *sc = cookie;
struct sun8i_crypto_rng *rng = &sc->sc_rng;
struct sun8i_crypto_task *task = rng->cr_task;
bool pending;
int error;
/*
* Test and set the RNG-pending flag. If it's already in
* progress, nothing to do here.
*/
mutex_enter(&sc->sc_lock);
pending = rng->cr_pending;
rng->cr_pending = true;
mutex_exit(&sc->sc_lock);
if (pending)
return;
/* Load the task descriptor. */
error = sun8i_crypto_task_load_trng(sc, task, SUN8I_CRYPTO_RNGBYTES);
if (error)
goto fail;
/* Submit! */
error = sun8i_crypto_submit(sc, task);
if (error)
goto fail;
/* All done! */
return;
fail: mutex_enter(&sc->sc_lock);
rng->cr_pending = false;
mutex_exit(&sc->sc_lock);
}
static void
sun8i_crypto_rng_done(struct sun8i_crypto_softc *sc,
struct sun8i_crypto_task *task, void *cookie, int error)
{
struct sun8i_crypto_rng *rng = cookie;
uint8_t *buf = rng->cr_buf.cb_kva;
uint32_t entropybits;
KASSERT(rng == &sc->sc_rng);
/* If anything went wrong, forget about it. */
if (error)
goto out;
/*
* This TRNG has quite low entropy at best. But if it fails a
* repeated output test, then assume it's busted.
*/
CTASSERT(SUN8I_CRYPTO_RNGBYTES <= UINT32_MAX/NBBY);
entropybits = (NBBY*SUN8I_CRYPTO_RNGBYTES)/SUN8I_CRYPTO_RNGENTROPY;
if (consttime_memequal(buf, buf + SUN8I_CRYPTO_RNGBYTES/2,
SUN8I_CRYPTO_RNGBYTES/2)) {
device_printf(sc->sc_dev, "failed repeated output test\n");
entropybits = 0;
}
/*
* Actually we don't believe in any of the entropy until this
* device has had more scrutiny.
*/
entropybits = 0;
/* Success! Enter and erase the data. */
rnd_add_data(&rng->cr_rndsource, buf, SUN8I_CRYPTO_RNGBYTES,
entropybits);
explicit_memset(buf, 0, SUN8I_CRYPTO_RNGBYTES);
out: /* Done -- clear the RNG-pending flag. */
mutex_enter(&sc->sc_lock);
rng->cr_pending = false;
mutex_exit(&sc->sc_lock);
}
/*
* Self-test
*/
static const uint8_t selftest_input[16];
static const uint8_t selftest_key[16];
static const uint8_t selftest_output[16] = {
0x66,0xe9,0x4b,0xd4,0xef,0x8a,0x2c,0x3b,
0x88,0x4c,0xfa,0x59,0xca,0x34,0x2b,0x2e,
};
static bool
sun8i_crypto_selftest(struct sun8i_crypto_softc *sc)
{
const size_t keybytes = sizeof selftest_key;
const size_t nbytes = sizeof selftest_input;
struct sun8i_crypto_selftest *selftest = &sc->sc_selftest;
struct sun8i_crypto_task *task;
unsigned timo = hztoms(SUN8I_CRYPTO_TIMEOUT);
uint32_t done, esr;
int error;
CTASSERT(sizeof selftest_input == sizeof selftest_output);
selftest->cs_pending = true;
/* Allocate an input buffer. */
error = sun8i_crypto_allocbuf(sc, nbytes, &selftest->cs_in,
BUS_DMA_WAITOK);
if (error)
goto fail0;
/* Allocate a key buffer. */
error = sun8i_crypto_allocbuf(sc, keybytes, &selftest->cs_key,
BUS_DMA_WAITOK);
if (error)
goto fail1;
/* Allocate an output buffer. */
error = sun8i_crypto_allocbuf(sc, nbytes, &selftest->cs_out,
BUS_DMA_WAITOK);
if (error)
goto fail2;
/* Allocate a task descriptor. */
task = selftest->cs_task = sun8i_crypto_task_get(sc,
sun8i_crypto_selftest_done, selftest, PR_WAITOK);
if (selftest->cs_task == NULL) {
error = ENOMEM;
goto fail3;
}
/* Copy the input and key into their buffers. */
memcpy(selftest->cs_in.cb_kva, selftest_input, nbytes);
memcpy(selftest->cs_key.cb_kva, selftest_key, keybytes);
/* Load the key, src, and dst for DMA transfers. */
error = bus_dmamap_load(sc->sc_dmat, task->ct_keymap,
selftest->cs_key.cb_kva, keybytes, NULL, BUS_DMA_WAITOK);
if (error)
goto fail4;
task->ct_flags |= TASK_KEY;
error = bus_dmamap_load(sc->sc_dmat, task->ct_srcmap,
selftest->cs_in.cb_kva, nbytes, NULL, BUS_DMA_WAITOK);
if (error)
goto fail5;
task->ct_flags |= TASK_SRC;
error = bus_dmamap_load(sc->sc_dmat, task->ct_dstmap,
selftest->cs_out.cb_kva, nbytes, NULL, BUS_DMA_WAITOK);
if (error)
goto fail6;
/* Set up the task descriptor. */
error = sun8i_crypto_task_load_aesecb(sc, task, nbytes,
SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128, SUN8I_CRYPTO_TDQC_OP_DIR_ENC);
if (error)
goto fail7;
/* Submit! */
error = sun8i_crypto_submit(sc, task);
if (error)
goto fail7;
device_printf(sc->sc_dev, "AES-128 self-test initiated\n");
/*
* Poll for completion. If not done after the timeout, force a
* timeout.
*/
for (; selftest->cs_pending && timo --> 0; DELAY(1000)) {
if (sun8i_crypto_poll(sc, &done, &esr))
(void)sun8i_crypto_done(sc, done, esr, getticks());
}
if (selftest->cs_pending) {
(void)sun8i_crypto_done(sc, 0, 0,
(unsigned)getticks() + SUN8I_CRYPTO_TIMEOUT);
KASSERT(!selftest->cs_pending);
}
return selftest->cs_passed;
fail7: bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap);
fail6: bus_dmamap_unload(sc->sc_dmat, task->ct_srcmap);
fail5: bus_dmamap_unload(sc->sc_dmat, task->ct_keymap);
fail4: sun8i_crypto_task_put(sc, task);
fail3: sun8i_crypto_freebuf(sc, nbytes, &selftest->cs_out);
fail2: sun8i_crypto_freebuf(sc, keybytes, &selftest->cs_key);
fail1: sun8i_crypto_freebuf(sc, nbytes, &selftest->cs_in);
fail0: aprint_error_dev(sc->sc_dev, "failed to run self-test, error=%d\n",
error);
selftest->cs_pending = false;
return false;
}
static bool
sun8i_crypto_selftest_check(struct sun8i_crypto_softc *sc, const char *title,
size_t n, const void *expected, const void *actual)
{
const uint8_t *e = expected;
const uint8_t *a = actual;
size_t i;
if (memcmp(e, a, n) == 0)
return true;
device_printf(sc->sc_dev, "self-test: %s\n", title);
printf("expected: ");
for (i = 0; i < n; i++)
printf("%02hhx", e[i]);
printf("\n");
printf("actual: ");
for (i = 0; i < n; i++)
printf("%02hhx", a[i]);
printf("\n");
return false;
}
static void
sun8i_crypto_selftest_done(struct sun8i_crypto_softc *sc,
struct sun8i_crypto_task *task, void *cookie, int error)
{
const size_t keybytes = sizeof selftest_key;
const size_t nbytes = sizeof selftest_input;
struct sun8i_crypto_selftest *selftest = cookie;
bool ok = true;
KASSERT(selftest == &sc->sc_selftest);
/* If anything went wrong, fail now. */
if (error) {
device_printf(sc->sc_dev, "self-test error=%d\n", error);
goto out;
}
/*
* Verify the input and key weren't clobbered, and verify the
* output matches what we expect.
*/
ok &= sun8i_crypto_selftest_check(sc, "input clobbered", nbytes,
selftest_input, selftest->cs_in.cb_kva);
ok &= sun8i_crypto_selftest_check(sc, "key clobbered", keybytes,
selftest_key, selftest->cs_key.cb_kva);
ok &= sun8i_crypto_selftest_check(sc, "output mismatch", nbytes,
selftest_output, selftest->cs_out.cb_kva);
selftest->cs_passed = ok;
if (ok)
device_printf(sc->sc_dev, "AES-128 self-test passed\n");
out: bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap);
bus_dmamap_unload(sc->sc_dmat, task->ct_srcmap);
bus_dmamap_unload(sc->sc_dmat, task->ct_keymap);
sun8i_crypto_task_put(sc, task);
sun8i_crypto_freebuf(sc, nbytes, &selftest->cs_out);
sun8i_crypto_freebuf(sc, keybytes, &selftest->cs_key);
sun8i_crypto_freebuf(sc, nbytes, &selftest->cs_in);
selftest->cs_pending = false;
}
/*
* Sysctl for testing
*/
struct sun8i_crypto_userreq {
kmutex_t cu_lock;
kcondvar_t cu_cv;
size_t cu_size;
struct sun8i_crypto_buf cu_buf;
struct sun8i_crypto_task *cu_task;
int cu_error;
bool cu_done;
bool cu_cancel;
};
static void
sun8i_crypto_sysctl_attach(struct sun8i_crypto_softc *sc)
{
struct sun8i_crypto_sysctl *cy = &sc->sc_sysctl;
int error;
/* hw.sun8icryptoN (node) */
error = sysctl_createv(&cy->cy_log, 0, NULL, &cy->cy_root_node,
CTLFLAG_PERMANENT, CTLTYPE_NODE, device_xname(sc->sc_dev),
SYSCTL_DESCR("sun8i crypto engine knobs"),
NULL, 0, NULL, 0,
CTL_HW, CTL_CREATE, CTL_EOL);
if (error) {
aprint_error_dev(sc->sc_dev,
"failed to set up sysctl hw.%s: %d\n",
device_xname(sc->sc_dev), error);
return;
}
/* hw.sun8icryptoN.rng (`struct', 4096-byte array) */
sysctl_createv(&cy->cy_log, 0, &cy->cy_root_node, &cy->cy_trng_node,
CTLFLAG_PERMANENT|CTLFLAG_READONLY|CTLFLAG_PRIVATE, CTLTYPE_STRUCT,
"rng", SYSCTL_DESCR("Read up to 4096 bytes out of the TRNG"),
&sun8i_crypto_sysctl_rng, 0, sc, 0, CTL_CREATE, CTL_EOL);
if (error) {
aprint_error_dev(sc->sc_dev,
"failed to set up sysctl hw.%s.rng: %d\n",
device_xname(sc->sc_dev), error);
return;
}
}
static int
sun8i_crypto_sysctl_rng(SYSCTLFN_ARGS)
{
struct sysctlnode node = *rnode;
struct sun8i_crypto_softc *sc = node.sysctl_data;
struct sun8i_crypto_userreq *req;
struct sun8i_crypto_task *task;
size_t size;
int error;
/* If oldp == NULL, the caller wants to learn the size. */
if (oldp == NULL) {
*oldlenp = 4096;
return 0;
}
/* Truncate to 4096 bytes. */
size = MIN(4096, *oldlenp);
if (size == 0)
return 0; /* nothing to do */
/* Allocate a request context. */
req = kmem_alloc(sizeof(*req), KM_NOSLEEP);
if (req == NULL)
return ENOMEM;
/* Initialize the request context. */
mutex_init(&req->cu_lock, MUTEX_DEFAULT, IPL_NONE);
cv_init(&req->cu_cv, "sun8isy");
req->cu_size = size;
req->cu_error = EIO;
req->cu_done = false;
req->cu_cancel = false;
/* Allocate a buffer for the RNG output. */
error = sun8i_crypto_allocbuf(sc, size, &req->cu_buf, BUS_DMA_NOWAIT);
if (error)
goto out0;
/* Allocate a task. */
task = req->cu_task = sun8i_crypto_task_get(sc,
sun8i_crypto_sysctl_rng_done, req, PR_NOWAIT);
if (task == NULL) {
error = ENOMEM;
goto out1;
}
/* Set the task up for TRNG to our buffer. */
error = bus_dmamap_load(sc->sc_dmat, task->ct_dstmap,
req->cu_buf.cb_kva, SUN8I_CRYPTO_RNGBYTES, NULL, BUS_DMA_NOWAIT);
if (error)
goto out2;
error = sun8i_crypto_task_load_trng(sc, task, SUN8I_CRYPTO_RNGBYTES);
if (error)
goto out3;
/* Submit! */
error = sun8i_crypto_submit(sc, task);
if (error) {
/* Make sure we don't restart the syscall -- just fail. */
if (error == ERESTART)
error = EBUSY;
goto out3;
}
/* Wait for the request to complete. */
mutex_enter(&req->cu_lock);
while (!req->cu_done) {
error = cv_wait_sig(&req->cu_cv, &req->cu_lock);
if (error) {
/*
* If we finished while waiting to acquire the
* lock, ignore the error and just return now.
* Otherwise, notify the callback that it has
* to clean up after us.
*/
if (req->cu_done)
error = 0;
else
req->cu_cancel = true;
break;
}
}
mutex_exit(&req->cu_lock);
/*
* Return early on error from cv_wait_sig, which means
* interruption; the callback will clean up instead.
*/
if (error)
return error;
/* Check for error from the device. */
error = req->cu_error;
if (error)
goto out3;
/* Copy out the data. */
node.sysctl_data = req->cu_buf.cb_kva;
node.sysctl_size = size;
error = sysctl_lookup(SYSCTLFN_CALL(&node));
/* Clear the buffer. */
explicit_memset(req->cu_buf.cb_kva, 0, size);
/* Clean up. */
out3: bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap);
out2: sun8i_crypto_task_put(sc, task);
out1: sun8i_crypto_freebuf(sc, req->cu_size, &req->cu_buf);
out0: cv_destroy(&req->cu_cv);
mutex_destroy(&req->cu_lock);
kmem_free(req, sizeof(*req));
return error;
}
static void
sun8i_crypto_sysctl_rng_done(struct sun8i_crypto_softc *sc,
struct sun8i_crypto_task *task, void *cookie, int error)
{
struct sun8i_crypto_userreq *req = cookie;
bool cancel;
/*
* Notify the waiting thread of the error, and find out whether
* that thread cancelled.
*/
mutex_enter(&req->cu_lock);
cancel = req->cu_cancel;
req->cu_error = error;
req->cu_done = true;
cv_broadcast(&req->cu_cv);
mutex_exit(&req->cu_lock);
/*
* If it wasn't cancelled, we're done -- the main thread will
* clean up after itself.
*/
if (!cancel)
return;
/* Clean up after the main thread cancelled. */
bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap);
sun8i_crypto_task_put(sc, task);
sun8i_crypto_freebuf(sc, req->cu_size, &req->cu_buf);
cv_destroy(&req->cu_cv);
mutex_destroy(&req->cu_lock);
kmem_free(req, sizeof(*req));
}
/*
* sun8i_crypto_register(sc)
*
* Register opencrypto algorithms supported by the crypto engine.
*/
static void
sun8i_crypto_register(struct sun8i_crypto_softc *sc)
{
struct sun8i_crypto_opencrypto *co = &sc->sc_opencrypto;
co->co_driverid = crypto_get_driverid(0);
if (co->co_driverid == (uint32_t)-1) {
aprint_error_dev(sc->sc_dev,
"failed to register crypto driver\n");
return;
}
sun8i_crypto_register1(sc, CRYPTO_AES_CBC);
sun8i_crypto_register1(sc, CRYPTO_AES_CTR);
#ifdef CRYPTO_AES_ECB
sun8i_crypto_register1(sc, CRYPTO_AES_ECB);
#endif
#ifdef CRYPTO_AES_XTS
sun8i_crypto_register1(sc, CRYPTO_AES_XTS);
#endif
#ifdef CRYPTO_DES_CBC
sun8i_crypto_register1(sc, CRYPTO_DES_CBC);
#endif
#ifdef CRYPTO_DES_ECB
sun8i_crypto_register1(sc, CRYPTO_DES_ECB);
#endif
sun8i_crypto_register1(sc, CRYPTO_3DES_CBC);
#ifdef CRYPTO_3DES_ECB
sun8i_crypto_register1(sc, CRYPTO_3DES_ECB);
#endif
sun8i_crypto_register1(sc, CRYPTO_MD5);
sun8i_crypto_register1(sc, CRYPTO_SHA1);
#ifdef CRYPTO_SHA224
sun8i_crypto_register1(sc, CRYPTO_SHA224);
#endif
#ifdef CRYPTO_SHA256
sun8i_crypto_register1(sc, CRYPTO_SHA256);
#endif
sun8i_crypto_register1(sc, CRYPTO_SHA1_HMAC);
sun8i_crypto_register1(sc, CRYPTO_SHA2_256_HMAC);
//sun8i_crypto_kregister(sc, CRK_MOD_EXP); /* XXX unclear */
}
/*
* sun8i_crypto_register1(sc, alg)
*
* Register support for one algorithm alg using
* sun8i_crypto_newsession/freesession/process.
*/
static void
sun8i_crypto_register1(struct sun8i_crypto_softc *sc, uint32_t alg)
{
crypto_register(sc->sc_opencrypto.co_driverid, alg, 0, 0,
sun8i_crypto_newsession,
sun8i_crypto_freesession,
sun8i_crypto_process,
sc);
}
/*
* sun8i_crypto_newsession(cookie, sidp, cri)
*
* Called by opencrypto to allocate a new session. We don't keep
* track of sessions, since there are no persistent keys in the
* hardware that we take advantage of, so this only validates the
* crypto operations and returns a dummy session id of 1.
*/
static int
sun8i_crypto_newsession(void *cookie, uint32_t *sidp, struct cryptoini *cri)
{
/* No composition of operations is supported here. */
if (cri->cri_next)
return EINVAL;
/*
* No variation of rounds is supported here. (XXX Unused and
* unimplemented in opencrypto(9) altogether?)
*/
if (cri->cri_rnd)
return EINVAL;
/*
* Validate per-algorithm key length.
*
* XXX Does opencrypto(9) do this internally?
*/
switch (cri->cri_alg) {
case CRYPTO_MD5:
case CRYPTO_SHA1:
#ifdef CRYPTO_SHA224
case CRYPTO_SHA224:
#endif
#ifdef CRYPTO_SHA256
case CRYPTO_SHA256:
#endif
if (cri->cri_klen)
return EINVAL;
break;
case CRYPTO_AES_CBC:
#ifdef CRYPTO_AES_ECB
case CRYPTO_AES_ECB:
#endif
switch (cri->cri_klen) {
case 128:
case 192:
case 256:
break;
default:
return EINVAL;
}
break;
case CRYPTO_AES_CTR:
/*
* opencrypto `AES-CTR' takes four bytes of the input
* block as the last four bytes of the key, for reasons
* that are not entirely clear.
*/
switch (cri->cri_klen) {
case 128 + 32:
case 192 + 32:
case 256 + 32:
break;
default:
return EINVAL;
}
break;
#ifdef CRYPTO_AES_XTS
case CRYPTO_AES_XTS:
switch (cri->cri_klen) {
case 256:
case 384:
case 512:
break;
default:
return EINVAL;
}
break;
#endif
case CRYPTO_DES_CBC:
#ifdef CRYPTO_DES_ECB
case CRYPTO_DES_ECB:
#endif
switch (cri->cri_klen) {
case 64:
break;
default:
return EINVAL;
}
break;
case CRYPTO_3DES_CBC:
#ifdef CRYPTO_3DES_ECB
case CRYPTO_3DES_ECB:
#endif
switch (cri->cri_klen) {
case 192:
break;
default:
return EINVAL;
}
break;
case CRYPTO_SHA1_HMAC:
/*
* XXX Unclear what the length limit is, but since HMAC
* behaves qualitatively different for a key of at
* least the full block size -- and is generally best
* to use with half the block size -- let's limit it to
* one block.
*/
if (cri->cri_klen % 8)
return EINVAL;
if (cri->cri_klen > 512)
return EINVAL;
break;
case CRYPTO_SHA2_256_HMAC:
if (cri->cri_klen % 8)
return EINVAL;
if (cri->cri_klen > 512)
return EINVAL;
break;
default:
panic("unsupported algorithm %d", cri->cri_alg);
}
KASSERT(cri->cri_klen % 8 == 0);
/* Success! */
*sidp = 1;
return 0;
}
/*
* sun8i_crypto_freesession(cookie, dsid)
*
* Called by opencrypto to free a session. We don't keep track of
* sessions, since there are no persistent keys in the hardware
* that we take advantage of, so this is a no-op.
*
* Note: dsid is actually a 64-bit quantity containing both the
* driver id in the high half and the session id in the low half.
*/
static void
sun8i_crypto_freesession(void *cookie, uint64_t dsid)
{
KASSERT((dsid & 0xffffffff) == 1);
}
/*
* sun8i_crypto_ivlen(crd)
*
* Return the crypto engine's notion of `IV length', in bytes, for
* an opencrypto operation.
*/
static u_int
sun8i_crypto_ivlen(const struct cryptodesc *crd)
{
switch (crd->crd_alg) {
case CRYPTO_AES_CBC:
return 16;
#ifdef CRYPTO_AES_XTS
case CRYPTO_AES_XTS:
return 16;
#endif
case CRYPTO_AES_CTR: /* XXX opencrypto quirk */
return 8;
#ifdef CRYPTO_DES_CBC
case CRYPTO_DES_CBC:
return 8;
#endif
case CRYPTO_3DES_CBC:
return 8;
case CRYPTO_MD5:
return 16;
#ifdef CRYPTO_SHA224
case CRYPTO_SHA224:
return 32;
#endif
#ifdef CRYPTO_SHA256
case CRYPTO_SHA256:
return 32;
#endif
case CRYPTO_SHA1_HMAC:
return 20;
case CRYPTO_SHA2_256_HMAC:
return 32;
default:
return 0;
}
}
/*
* sun8i_crypto_process(cookie, crp, hint)
*
* Main opencrypto processing dispatch.
*/
static int
sun8i_crypto_process(void *cookie, struct cryptop *crp, int hint)
{
struct sun8i_crypto_softc *sc = cookie;
struct sun8i_crypto_task *task;
struct cryptodesc *crd = crp->crp_desc;
unsigned klen, ivlen;
uint32_t tdqc = 0, tdqs = 0;
uint32_t dir, method, mode = 0, ctrwidth = 0, aeskeysize = 0;
const uint32_t tdqa = 0;
int error;
SDT_PROBE3(sdt, sun8i_crypto, process, entry, sc, crp, hint);
/* Reject compositions -- we do not handle them. */
if (crd->crd_next != NULL) {
error = EOPNOTSUPP;
goto fail0;
}
/* Reject transfers with nonsense skip. */
if (crd->crd_skip < 0) {
error = EINVAL;
goto fail0;
}
/*
* Actually just reject any nonzero skip, because it requires
* DMA segment bookkeeping that we don't do yet.
*/
if (crd->crd_skip) {
error = EOPNOTSUPP;
goto fail0;
}
/* Reject large transfers. */
if (crd->crd_len > SUN8I_CRYPTO_MAXDMASIZE) {
error = EFBIG;
goto fail0;
}
/* Reject nonsense, unaligned, or mismatched lengths. */
if (crd->crd_len < 0 ||
crd->crd_len % 4 ||
crd->crd_len != crp->crp_ilen) {
error = EINVAL;
goto fail0;
}
/* Reject mismatched buffer lengths. */
/* XXX Handle crd_skip. */
if (crp->crp_flags & CRYPTO_F_IMBUF) {
struct mbuf *m = crp->crp_buf;
uint32_t nbytes = 0;
while (m != NULL) {
KASSERT(m->m_len >= 0);
if (m->m_len > crd->crd_len ||
nbytes > crd->crd_len - m->m_len) {
error = EINVAL;
goto fail0;
}
nbytes += m->m_len;
m = m->m_next;
}
if (nbytes != crd->crd_len) {
error = EINVAL;
goto fail0;
}
} else if (crp->crp_flags & CRYPTO_F_IOV) {
struct uio *uio = crp->crp_buf;
if (uio->uio_resid != crd->crd_len) {
error = EINVAL;
goto fail0;
}
}
/* Get a task, or fail with ERESTART if we can't. */
task = sun8i_crypto_task_get(sc, &sun8i_crypto_callback, crp,
PR_NOWAIT);
if (task == NULL) {
/*
* Don't invoke crypto_done -- we are asking the
* opencrypto(9) machinery to queue the request and get
* back to us.
*/
SDT_PROBE3(sdt, sun8i_crypto, process, busy, sc, crp, hint);
return ERESTART;
}
/* Load key in, if relevant. */
klen = crd->crd_klen;
if (klen) {
if (crd->crd_alg == CRYPTO_AES_CTR)
/* AES-CTR is special -- see IV processing below. */
klen -= 32;
error = bus_dmamap_load(sc->sc_dmat, task->ct_keymap,
crd->crd_key, klen/8, NULL, BUS_DMA_NOWAIT);
if (error)
goto fail1;
task->ct_flags |= TASK_KEY;
}
/* Handle the IV, if relevant. */
ivlen = sun8i_crypto_ivlen(crd);
if (ivlen) {
void *iv;
/*
* If there's an explicit IV, use it; otherwise
* randomly generate one.
*/
if (crd->crd_flags & CRD_F_IV_EXPLICIT) {
iv = crd->crd_iv;
} else {
cprng_fast(task->ct_iv, ivlen);
iv = task->ct_iv;
}
/*
* If the IV is not already present in the user's
* buffer, copy it over.
*/
if ((crd->crd_flags & CRD_F_IV_PRESENT) == 0) {
if (crp->crp_flags & CRYPTO_F_IMBUF) {
m_copyback(crp->crp_buf, crd->crd_inject,
ivlen, iv);
} else if (crp->crp_flags & CRYPTO_F_IOV) {
cuio_copyback(crp->crp_buf, crd->crd_inject,
ivlen, iv);
} else {
panic("invalid buffer type %x",
crp->crp_flags);
}
}
/*
* opencrypto's idea of `AES-CTR' is special.
*
* - The low 4 bytes of the input block are drawn from
* an extra 4 bytes at the end of the key.
*
* - The next 8 bytes of the input block are drawn from
* the opencrypto iv.
*
* - The high 4 bytes are the big-endian block counter,
* which starts at 1 because why not.
*/
if (crd->crd_alg == CRYPTO_AES_CTR) {
uint8_t block[16];
uint32_t blkno = 1;
/* Format the initial input block. */
memcpy(block, crd->crd_key + klen/8, 4);
memcpy(block + 4, iv, 8);
be32enc(block + 12, blkno);
/* Copy it into the DMA buffer. */
memcpy(task->ct_iv, block, 16);
iv = task->ct_iv;
ivlen = 16;
}
/* Load the IV. */
error = bus_dmamap_load(sc->sc_dmat, task->ct_ivmap, iv, ivlen,
NULL, BUS_DMA_NOWAIT);
if (error)
goto fail1;
task->ct_flags |= TASK_IV;
}
/* Load the src and dst. */
if (crp->crp_flags & CRYPTO_F_IMBUF) {
struct mbuf *m = crp->crp_buf;
/* XXX Handle crd_skip. */
KASSERT(crd->crd_skip == 0);
error = bus_dmamap_load_mbuf(sc->sc_dmat, task->ct_srcmap, m,
BUS_DMA_NOWAIT);
if (error)
goto fail1;
task->ct_flags |= TASK_SRC;
/* XXX Handle crd_skip. */
KASSERT(crd->crd_skip == 0);
error = bus_dmamap_load_mbuf(sc->sc_dmat, task->ct_dstmap, m,
BUS_DMA_NOWAIT);
if (error)
goto fail1;
} else if (crp->crp_flags & CRYPTO_F_IOV) {
struct uio *uio = crp->crp_buf;
/* XXX Handle crd_skip. */
KASSERT(crd->crd_skip == 0);
error = bus_dmamap_load_uio(sc->sc_dmat, task->ct_srcmap, uio,
BUS_DMA_NOWAIT);
if (error)
goto fail1;
task->ct_flags |= TASK_SRC;
/* XXX Handle crd_skip. */
KASSERT(crd->crd_skip == 0);
error = bus_dmamap_load_uio(sc->sc_dmat, task->ct_dstmap, uio,
BUS_DMA_NOWAIT);
if (error)
goto fail1;
} else {
panic("invalid buffer type %x", crp->crp_flags);
}
/* Set the encryption direction. */
if (crd->crd_flags & CRD_F_ENCRYPT)
dir = SUN8I_CRYPTO_TDQC_OP_DIR_ENC;
else
dir = SUN8I_CRYPTO_TDQC_OP_DIR_DEC;
tdqc |= __SHIFTIN(dir, SUN8I_CRYPTO_TDQC_OP_DIR);
/* Set the method. */
switch (crd->crd_alg) {
case CRYPTO_AES_CBC:
case CRYPTO_AES_CTR:
#ifdef CRYPTO_AES_ECB
case CRYPTO_AES_ECB:
#endif
method = SUN8I_CRYPTO_TDQC_METHOD_AES;
break;
#ifdef CRYPTO_AES_XTS
case CRYPTO_AES_XTS:
method = SUN8I_CRYPTO_TDQC_METHOD_AES;
break;
#endif
case CRYPTO_DES_CBC:
#ifdef CRYPTO_DES_ECB
case CRYPTO_DES_ECB:
#endif
method = SUN8I_CRYPTO_TDQC_METHOD_DES;
break;
case CRYPTO_3DES_CBC:
#ifdef CRYPTO_3DES_ECB
case CRYPTO_3DES_ECB:
#endif
method = SUN8I_CRYPTO_TDQC_METHOD_3DES;
break;
case CRYPTO_MD5:
method = SUN8I_CRYPTO_TDQC_METHOD_MD5;
break;
case CRYPTO_SHA1:
method = SUN8I_CRYPTO_TDQC_METHOD_SHA1;
break;
#ifdef CRYPTO_SHA224
case CRYPTO_SHA224:
method = SUN8I_CRYPTO_TDQC_METHOD_SHA224;
break;
#endif
#ifdef CRYPTO_SHA256
case CRYPTO_SHA256:
method = SUN8I_CRYPTO_TDQC_METHOD_SHA256;
break;
#endif
case CRYPTO_SHA1_HMAC:
method = SUN8I_CRYPTO_TDQC_METHOD_HMAC_SHA1;
break;
case CRYPTO_SHA2_256_HMAC:
method = SUN8I_CRYPTO_TDQC_METHOD_HMAC_SHA256;
break;
default:
panic("unknown algorithm %d", crd->crd_alg);
}
tdqc |= __SHIFTIN(method, SUN8I_CRYPTO_TDQC_METHOD);
/* Set the key selector. No idea how to use the internal keys. */
tdqs |= __SHIFTIN(SUN8I_CRYPTO_TDQS_SKEY_SELECT_SS_KEYx,
SUN8I_CRYPTO_TDQS_SKEY_SELECT);
/* XXX Deal with AES_CTS_Last_Block_Flag. */
/* Set the mode. */
switch (crd->crd_alg) {
#ifdef CRYPTO_AES_ECB
case CRYPTO_AES_ECB:
mode = SUN8I_CRYPTO_TDQS_OP_MODE_ECB;
break;
#endif
#ifdef CRYPTO_DES_ECB
case CRYPTO_DES_ECB:
mode = SUN8I_CRYPTO_TDQS_OP_MODE_ECB;
break;
#endif
#ifdef CRYPTO_3DES_ECB
case CRYPTO_3DES_ECB:
mode = SUN8I_CRYPTO_TDQS_OP_MODE_ECB;
break;
#endif
case CRYPTO_AES_CBC:
case CRYPTO_DES_CBC:
case CRYPTO_3DES_CBC:
mode = SUN8I_CRYPTO_TDQS_OP_MODE_CBC;
break;
case CRYPTO_AES_CTR:
mode = SUN8I_CRYPTO_TDQS_OP_MODE_CTR;
break;
#ifdef CRYPTO_AES_XTS
case CRYPTO_AES_XTS:
mode = SUN8I_CRYPTO_TDQS_OP_MODE_CTS;
break;
#endif
default:
panic("unknown algorithm %d", crd->crd_alg);
}
tdqs |= __SHIFTIN(mode, SUN8I_CRYPTO_TDQS_OP_MODE);
/* Set the CTR width. */
switch (crd->crd_alg) {
case CRYPTO_AES_CTR:
ctrwidth = SUN8I_CRYPTO_TDQS_CTR_WIDTH_32;
break;
}
tdqs |= __SHIFTIN(ctrwidth, SUN8I_CRYPTO_TDQS_CTR_WIDTH);
/* Set the AES key size. */
switch (crd->crd_alg) {
case CRYPTO_AES_CBC:
#ifdef CRYPTO_AES_ECB
case CRYPTO_AES_ECB:
#endif
switch (crd->crd_klen) {
case 128:
aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128;
break;
case 192:
aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_192;
break;
case 256:
aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_256;
break;
default:
panic("invalid AES key size in bits: %u",
crd->crd_klen);
}
break;
case CRYPTO_AES_CTR:
switch (crd->crd_klen) {
case 128 + 32:
aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128;
break;
case 192 + 32:
aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_192;
break;
case 256 + 32:
aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_256;
break;
default:
panic("invalid `AES-CTR' ` ``key'' size' in bits: %u",
crd->crd_klen);
}
break;
#ifdef CRYPTO_AES_XTS
case CRYPTO_AES_XTS:
switch (crd->crd_klen) {
case 256:
aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128;
break;
case 384:
aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_192;
break;
case 512:
aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_256;
break;
default:
panic("invalid AES-XTS key size in bits: %u",
crd->crd_klen);
}
break;
#endif
}
tdqs |= __SHIFTIN(aeskeysize, SUN8I_CRYPTO_TDQS_AES_KEYSIZE);
/* Set up the task descriptor. */
error = sun8i_crypto_task_load(sc, task, crd->crd_len,
tdqc, tdqs, tdqa);
if (error)
goto fail2;
/* Submit! */
error = sun8i_crypto_submit(sc, task);
if (error)
goto fail2;
/* Success! */
SDT_PROBE4(sdt, sun8i_crypto, process, queued, sc, crp, hint, task);
return 0;
fail2: bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap);
fail1: if (task->ct_flags & TASK_SRC)
bus_dmamap_unload(sc->sc_dmat, task->ct_srcmap);
if (task->ct_flags & TASK_CTR)
bus_dmamap_unload(sc->sc_dmat, task->ct_ctrmap);
if (task->ct_flags & TASK_IV)
bus_dmamap_unload(sc->sc_dmat, task->ct_ivmap);
if (task->ct_flags & TASK_KEY)
bus_dmamap_unload(sc->sc_dmat, task->ct_keymap);
sun8i_crypto_task_put(sc, task);
fail0: KASSERT(error);
KASSERT(error != ERESTART);
crp->crp_etype = error;
SDT_PROBE3(sdt, sun8i_crypto, process, done, sc, crp, error);
crypto_done(crp);
return 0;
}
/*
* sun8i_crypto_callback(sc, task, cookie, error)
*
* Completion callback for a task submitted via opencrypto.
* Release the task and pass the error on to opencrypto with
* crypto_done.
*/
static void
sun8i_crypto_callback(struct sun8i_crypto_softc *sc,
struct sun8i_crypto_task *task, void *cookie, int error)
{
struct cryptop *crp = cookie;
struct cryptodesc *crd __diagused = crp->crp_desc;
KASSERT(error != ERESTART);
KASSERT(crd != NULL);
KASSERT(crd->crd_next == NULL);
/* Return the number of bytes processed. */
crp->crp_olen = error ? 0 : crp->crp_ilen;
bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap);
bus_dmamap_unload(sc->sc_dmat, task->ct_srcmap);
if (task->ct_flags & TASK_CTR)
bus_dmamap_unload(sc->sc_dmat, task->ct_ctrmap);
if (task->ct_flags & TASK_IV)
bus_dmamap_unload(sc->sc_dmat, task->ct_ivmap);
if (task->ct_flags & TASK_KEY)
bus_dmamap_unload(sc->sc_dmat, task->ct_keymap);
sun8i_crypto_task_put(sc, task);
KASSERT(error != ERESTART);
crp->crp_etype = error;
SDT_PROBE3(sdt, sun8i_crypto, process, done, sc, crp, error);
crypto_done(crp);
}