Lines Matching defs:crp
215 static int crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint);
217 static void crypto_batch_enqueue(struct cryptop *crp);
1232 ("incoming crp with invalid %s buffer type", name));
1236 ("incoming crp with -ve %s buffer length", name));
1240 ("incoming crp uses dmap on supported arch"));
1242 ("incoming crp with -ve %s buffer length", name));
1244 ("incoming crp with -ve %s buffer offset", name));
1246 ("incoming crp with %s buffer offset greater than page size"
1255 crp_sanity(struct cryptop *crp)
1261 KASSERT(crp->crp_session != NULL, ("incoming crp without a session"));
1262 KASSERT(crp->crp_obuf.cb_type >= CRYPTO_BUF_NONE &&
1263 crp->crp_obuf.cb_type <= CRYPTO_BUF_LAST,
1264 ("incoming crp with invalid output buffer type"));
1265 KASSERT(crp->crp_etype == 0, ("incoming crp with error"));
1266 KASSERT(!(crp->crp_flags & CRYPTO_F_DONE),
1267 ("incoming crp already done"));
1269 csp = &crp->crp_session->csp;
1270 cb_sanity(&crp->crp_buf, "input");
1271 ilen = crypto_buffer_len(&crp->crp_buf);
1275 if (crp->crp_obuf.cb_type != CRYPTO_BUF_NONE) {
1276 cb_sanity(&crp->crp_obuf, "output");
1277 out = &crp->crp_obuf;
1281 KASSERT(crp->crp_obuf.cb_type == CRYPTO_BUF_NONE,
1282 ("incoming crp with separate output buffer "
1287 KASSERT(crp->crp_op == CRYPTO_OP_COMPRESS ||
1288 crp->crp_op == CRYPTO_OP_DECOMPRESS,
1289 ("invalid compression op %x", crp->crp_op));
1292 KASSERT(crp->crp_op == CRYPTO_OP_ENCRYPT ||
1293 crp->crp_op == CRYPTO_OP_DECRYPT,
1294 ("invalid cipher op %x", crp->crp_op));
1297 KASSERT(crp->crp_op == CRYPTO_OP_COMPUTE_DIGEST ||
1298 crp->crp_op == CRYPTO_OP_VERIFY_DIGEST,
1299 ("invalid digest op %x", crp->crp_op));
1302 KASSERT(crp->crp_op ==
1304 crp->crp_op ==
1306 ("invalid AEAD op %x", crp->crp_op));
1307 KASSERT(crp->crp_flags & CRYPTO_F_IV_SEPARATE,
1311 KASSERT(crp->crp_op ==
1313 crp->crp_op ==
1315 ("invalid ETA op %x", crp->crp_op));
1319 if (crp->crp_aad == NULL) {
1320 KASSERT(crp->crp_aad_start == 0 ||
1321 crp->crp_aad_start < ilen,
1323 KASSERT(crp->crp_aad_length != 0 ||
1324 crp->crp_aad_start == 0,
1326 KASSERT(crp->crp_aad_length == 0 ||
1327 crp->crp_aad_start + crp->crp_aad_length <= ilen,
1332 KASSERT(crp->crp_aad_start == 0,
1334 KASSERT(crp->crp_aad_length != 0,
1338 KASSERT(crp->crp_aad == NULL && crp->crp_aad_start == 0 &&
1339 crp->crp_aad_length == 0,
1343 KASSERT((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0,
1345 KASSERT(crp->crp_iv_start == 0,
1347 } else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE) {
1348 KASSERT(crp->crp_iv_start == 0,
1351 KASSERT(crp->crp_iv_start < ilen,
1353 KASSERT(crp->crp_iv_start + csp->csp_ivlen <= ilen,
1357 KASSERT(crp->crp_payload_start == 0 ||
1358 crp->crp_payload_start < ilen,
1360 KASSERT(crp->crp_payload_start + crp->crp_payload_length <=
1363 KASSERT(crp->crp_payload_output_start == 0,
1366 KASSERT(!(crp->crp_op & CRYPTO_OP_VERIFY_DIGEST),
1368 KASSERT(crp->crp_payload_output_start == 0,
1371 KASSERT(crp->crp_payload_output_start == 0 ||
1372 crp->crp_payload_output_start < olen,
1374 KASSERT(crp->crp_payload_output_start +
1375 crp->crp_payload_length <= olen,
1380 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST)
1384 KASSERT(crp->crp_digest_start == 0 ||
1385 crp->crp_digest_start < len,
1388 KASSERT(crp->crp_digest_start + csp->csp_auth_mlen <= len,
1391 KASSERT(crp->crp_digest_start == 0,
1396 crp->crp_cipher_key != NULL,
1399 KASSERT(csp->csp_auth_key != NULL || crp->crp_auth_key != NULL,
1401 KASSERT(crp->crp_callback != NULL, ("incoming crp without callback"));
1406 crypto_dispatch_one(struct cryptop *crp, int hint)
1412 crp_sanity(crp);
1416 crp->crp_retw_id = crp->crp_session->id % crypto_workers_num;
1423 cap = crp->crp_session->cap;
1425 result = crypto_invoke(cap, crp, hint);
1434 crypto_batch_enqueue(crp);
1439 crypto_dispatch(struct cryptop *crp)
1441 return (crypto_dispatch_one(crp, 0));
1445 crypto_dispatch_async(struct cryptop *crp, int flags)
1449 if (!CRYPTO_SESS_SYNC(crp->crp_session)) {
1454 return (crypto_dispatch(crp));
1458 crp_sanity(crp);
1462 crp->crp_retw_id = crp->crp_session->id % crypto_workers_num;
1464 crp->crp_flags |= CRYPTO_F_ASYNC_ORDERED;
1465 ret_worker = CRYPTO_RETW(crp->crp_retw_id);
1467 crp->crp_seq = ret_worker->reorder_ops++;
1470 TASK_INIT(&crp->crp_task, 0, crypto_task_invoke, crp);
1471 taskqueue_enqueue(crypto_tq, &crp->crp_task);
1478 struct cryptop *crp;
1481 while ((crp = TAILQ_FIRST(crpq)) != NULL) {
1482 hint = TAILQ_NEXT(crp, crp_next) != NULL ? CRYPTO_HINT_MORE : 0;
1483 TAILQ_REMOVE(crpq, crp, crp_next);
1484 if (crypto_dispatch_one(crp, hint) != 0)
1485 crypto_batch_enqueue(crp);
1490 crypto_batch_enqueue(struct cryptop *crp)
1494 TAILQ_INSERT_TAIL(&crp_q, crp, crp_next);
1504 struct cryptop *crp;
1507 crp = (struct cryptop *)ctx;
1508 cap = crp->crp_session->cap;
1509 result = crypto_invoke(cap, crp, 0);
1511 crypto_batch_enqueue(crp);
1518 crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint)
1522 KASSERT(crp != NULL, ("%s: crp == NULL", __func__));
1523 KASSERT(crp->crp_callback != NULL,
1524 ("%s: crp->crp_callback == NULL", __func__));
1525 KASSERT(crp->crp_session != NULL,
1526 ("%s: crp->crp_session == NULL", __func__));
1546 * Perhaps provide a method to allow a crp's session to
1549 csp = crp->crp_session->csp;
1550 crypto_freesession(crp->crp_session);
1565 crp->crp_session = nses;
1567 crp->crp_etype = EAGAIN;
1568 crypto_done(crp);
1576 error = CRYPTODEV_PROCESS(cap->cc_dev, crp, hint);
1585 crypto_destroyreq(struct cryptop *crp)
1597 KASSERT(crp2 != crp,
1599 crp));
1606 KASSERT(crp2 != crp,
1608 crp));
1617 crypto_freereq(struct cryptop *crp)
1619 if (crp == NULL)
1622 crypto_destroyreq(crp);
1623 uma_zfree(cryptop_zone, crp);
1627 crypto_initreq(struct cryptop *crp, crypto_session_t cses)
1629 memset(crp, 0, sizeof(*crp));
1630 crp->crp_session = cses;
1636 struct cryptop *crp;
1639 crp = uma_zalloc(cryptop_zone, how);
1640 if (crp != NULL)
1641 crypto_initreq(crp, cses);
1642 return (crp);
1652 crypto_clonereq(struct cryptop *crp, crypto_session_t cses, int how)
1656 MPASS((crp->crp_flags & CRYPTO_F_DONE) == 0);
1661 memcpy(&new->crp_startcopy, &crp->crp_startcopy,
1670 crypto_done(struct cryptop *crp)
1672 KASSERT((crp->crp_flags & CRYPTO_F_DONE) == 0,
1673 ("crypto_done: op already done, flags 0x%x", crp->crp_flags));
1674 crp->crp_flags |= CRYPTO_F_DONE;
1675 if (crp->crp_etype != 0)
1685 if ((crp->crp_flags & CRYPTO_F_ASYNC_ORDERED) == 0 &&
1686 ((crp->crp_flags & CRYPTO_F_CBIMM) != 0 ||
1687 ((crp->crp_flags & CRYPTO_F_CBIFSYNC) != 0 &&
1688 CRYPTO_SESS_SYNC(crp->crp_session)))) {
1694 crp->crp_callback(crp);
1699 ret_worker = CRYPTO_RETW(crp->crp_retw_id);
1705 if ((crp->crp_flags & CRYPTO_F_ASYNC_ORDERED) != 0) {
1711 if (CRYPTO_SEQ_GT(crp->crp_seq, tmp->crp_seq)) {
1714 crp, crp_next);
1720 &ret_worker->crp_ordered_ret_q, crp,
1724 wake = crp->crp_seq == ret_worker->reorder_cur_seq;
1727 TAILQ_INSERT_TAIL(&ret_worker->crp_ret_q, crp,
1761 struct cryptop *crp, *submit;
1778 TAILQ_FOREACH(crp, &crp_q, crp_next) {
1779 cap = crp->crp_session->cap;
1789 submit = crp;
1805 submit = crp;
1940 struct cryptop *crp;
1949 TAILQ_FOREACH(crp, &crp_q, crp_next) {
1951 , crp->crp_session->cap->cc_hid
1952 , (int) crypto_ses2caps(crp->crp_session)
1953 , crp->crp_olen
1954 , crp->crp_etype
1955 , crp->crp_flags
1956 , device_get_nameunit(crp->crp_session->cap->cc_dev)
1957 , crp->crp_callback
1964 TAILQ_FOREACH(crp, &ret_worker->crp_ret_q, crp_next) {
1967 , crp->crp_session->cap->cc_hid
1968 , crp->crp_etype
1969 , crp->crp_flags
1970 , crp->crp_callback