1// SPDX-License-Identifier: GPL-2.0 2/* Copyright (c) 2019 HiSilicon Limited. */ 3#include <crypto/internal/acompress.h> 4#include <linux/bitfield.h> 5#include <linux/bitmap.h> 6#include <linux/dma-mapping.h> 7#include <linux/scatterlist.h> 8#include "zip.h" 9 10/* hisi_zip_sqe dw3 */ 11#define HZIP_BD_STATUS_M GENMASK(7, 0) 12/* hisi_zip_sqe dw7 */ 13#define HZIP_IN_SGE_DATA_OFFSET_M GENMASK(23, 0) 14#define HZIP_SQE_TYPE_M GENMASK(31, 28) 15/* hisi_zip_sqe dw8 */ 16#define HZIP_OUT_SGE_DATA_OFFSET_M GENMASK(23, 0) 17/* hisi_zip_sqe dw9 */ 18#define HZIP_REQ_TYPE_M GENMASK(7, 0) 19#define HZIP_ALG_TYPE_ZLIB 0x02 20#define HZIP_ALG_TYPE_GZIP 0x03 21#define HZIP_BUF_TYPE_M GENMASK(11, 8) 22#define HZIP_PBUFFER 0x0 23#define HZIP_SGL 0x1 24 25#define HZIP_ZLIB_HEAD_SIZE 2 26#define HZIP_GZIP_HEAD_SIZE 10 27 28#define GZIP_HEAD_FHCRC_BIT BIT(1) 29#define GZIP_HEAD_FEXTRA_BIT BIT(2) 30#define GZIP_HEAD_FNAME_BIT BIT(3) 31#define GZIP_HEAD_FCOMMENT_BIT BIT(4) 32 33#define GZIP_HEAD_FLG_SHIFT 3 34#define GZIP_HEAD_FEXTRA_SHIFT 10 35#define GZIP_HEAD_FEXTRA_XLEN 2UL 36#define GZIP_HEAD_FHCRC_SIZE 2 37 38#define HZIP_GZIP_HEAD_BUF 256 39#define HZIP_ALG_PRIORITY 300 40#define HZIP_SGL_SGE_NR 10 41 42#define HZIP_ALG_ZLIB GENMASK(1, 0) 43#define HZIP_ALG_GZIP GENMASK(3, 2) 44 45static const u8 zlib_head[HZIP_ZLIB_HEAD_SIZE] = {0x78, 0x9c}; 46static const u8 gzip_head[HZIP_GZIP_HEAD_SIZE] = { 47 0x1f, 0x8b, 0x08, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x03 48}; 49 50enum hisi_zip_alg_type { 51 HZIP_ALG_TYPE_COMP = 0, 52 HZIP_ALG_TYPE_DECOMP = 1, 53}; 54 55enum { 56 HZIP_QPC_COMP, 57 HZIP_QPC_DECOMP, 58 HZIP_CTX_Q_NUM 59}; 60 61#define COMP_NAME_TO_TYPE(alg_name) \ 62 (!strcmp((alg_name), "zlib-deflate") ? HZIP_ALG_TYPE_ZLIB : \ 63 !strcmp((alg_name), "gzip") ? HZIP_ALG_TYPE_GZIP : 0) \ 64 65#define TO_HEAD_SIZE(req_type) \ 66 (((req_type) == HZIP_ALG_TYPE_ZLIB) ? sizeof(zlib_head) : \ 67 ((req_type) == HZIP_ALG_TYPE_GZIP) ? sizeof(gzip_head) : 0) \ 68 69#define TO_HEAD(req_type) \ 70 (((req_type) == HZIP_ALG_TYPE_ZLIB) ? zlib_head : \ 71 ((req_type) == HZIP_ALG_TYPE_GZIP) ? gzip_head : NULL) \ 72 73struct hisi_zip_req { 74 struct acomp_req *req; 75 u32 sskip; 76 u32 dskip; 77 struct hisi_acc_hw_sgl *hw_src; 78 struct hisi_acc_hw_sgl *hw_dst; 79 dma_addr_t dma_src; 80 dma_addr_t dma_dst; 81 u16 req_id; 82}; 83 84struct hisi_zip_req_q { 85 struct hisi_zip_req *q; 86 unsigned long *req_bitmap; 87 rwlock_t req_lock; 88 u16 size; 89}; 90 91struct hisi_zip_qp_ctx { 92 struct hisi_qp *qp; 93 struct hisi_zip_req_q req_q; 94 struct hisi_acc_sgl_pool *sgl_pool; 95 struct hisi_zip *zip_dev; 96 struct hisi_zip_ctx *ctx; 97}; 98 99struct hisi_zip_sqe_ops { 100 u8 sqe_type; 101 void (*fill_addr)(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req); 102 void (*fill_buf_size)(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req); 103 void (*fill_buf_type)(struct hisi_zip_sqe *sqe, u8 buf_type); 104 void (*fill_req_type)(struct hisi_zip_sqe *sqe, u8 req_type); 105 void (*fill_tag)(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req); 106 void (*fill_sqe_type)(struct hisi_zip_sqe *sqe, u8 sqe_type); 107 u32 (*get_tag)(struct hisi_zip_sqe *sqe); 108 u32 (*get_status)(struct hisi_zip_sqe *sqe); 109 u32 (*get_dstlen)(struct hisi_zip_sqe *sqe); 110}; 111 112struct hisi_zip_ctx { 113 struct hisi_zip_qp_ctx qp_ctx[HZIP_CTX_Q_NUM]; 114 const struct hisi_zip_sqe_ops *ops; 115}; 116 117static int sgl_sge_nr_set(const char *val, const struct kernel_param *kp) 118{ 119 int ret; 120 u16 n; 121 122 if (!val) 123 return -EINVAL; 124 125 ret = kstrtou16(val, 10, &n); 126 if (ret || n == 0 || n > HISI_ACC_SGL_SGE_NR_MAX) 127 return -EINVAL; 128 129 return param_set_ushort(val, kp); 130} 131 132static const struct kernel_param_ops sgl_sge_nr_ops = { 133 .set = sgl_sge_nr_set, 134 .get = param_get_ushort, 135}; 136 137static u16 sgl_sge_nr = HZIP_SGL_SGE_NR; 138module_param_cb(sgl_sge_nr, &sgl_sge_nr_ops, &sgl_sge_nr, 0444); 139MODULE_PARM_DESC(sgl_sge_nr, "Number of sge in sgl(1-255)"); 140 141static u32 get_extra_field_size(const u8 *start) 142{ 143 return *((u16 *)start) + GZIP_HEAD_FEXTRA_XLEN; 144} 145 146static u32 get_name_field_size(const u8 *start) 147{ 148 return strlen(start) + 1; 149} 150 151static u32 get_comment_field_size(const u8 *start) 152{ 153 return strlen(start) + 1; 154} 155 156static u32 __get_gzip_head_size(const u8 *src) 157{ 158 u8 head_flg = *(src + GZIP_HEAD_FLG_SHIFT); 159 u32 size = GZIP_HEAD_FEXTRA_SHIFT; 160 161 if (head_flg & GZIP_HEAD_FEXTRA_BIT) 162 size += get_extra_field_size(src + size); 163 if (head_flg & GZIP_HEAD_FNAME_BIT) 164 size += get_name_field_size(src + size); 165 if (head_flg & GZIP_HEAD_FCOMMENT_BIT) 166 size += get_comment_field_size(src + size); 167 if (head_flg & GZIP_HEAD_FHCRC_BIT) 168 size += GZIP_HEAD_FHCRC_SIZE; 169 170 return size; 171} 172 173static u32 __maybe_unused get_gzip_head_size(struct scatterlist *sgl) 174{ 175 char buf[HZIP_GZIP_HEAD_BUF]; 176 177 sg_copy_to_buffer(sgl, sg_nents(sgl), buf, sizeof(buf)); 178 179 return __get_gzip_head_size(buf); 180} 181 182static int add_comp_head(struct scatterlist *dst, u8 req_type) 183{ 184 int head_size = TO_HEAD_SIZE(req_type); 185 const u8 *head = TO_HEAD(req_type); 186 int ret; 187 188 ret = sg_copy_from_buffer(dst, sg_nents(dst), head, head_size); 189 if (unlikely(ret != head_size)) { 190 pr_err("the head size of buffer is wrong (%d)!\n", ret); 191 return -ENOMEM; 192 } 193 194 return head_size; 195} 196 197static int get_comp_head_size(struct acomp_req *acomp_req, u8 req_type) 198{ 199 if (unlikely(!acomp_req->src || !acomp_req->slen)) 200 return -EINVAL; 201 202 if (unlikely(req_type == HZIP_ALG_TYPE_GZIP && 203 acomp_req->slen < GZIP_HEAD_FEXTRA_SHIFT)) 204 return -EINVAL; 205 206 switch (req_type) { 207 case HZIP_ALG_TYPE_ZLIB: 208 return TO_HEAD_SIZE(HZIP_ALG_TYPE_ZLIB); 209 case HZIP_ALG_TYPE_GZIP: 210 return TO_HEAD_SIZE(HZIP_ALG_TYPE_GZIP); 211 default: 212 pr_err("request type does not support!\n"); 213 return -EINVAL; 214 } 215} 216 217static struct hisi_zip_req *hisi_zip_create_req(struct acomp_req *req, 218 struct hisi_zip_qp_ctx *qp_ctx, 219 size_t head_size, bool is_comp) 220{ 221 struct hisi_zip_req_q *req_q = &qp_ctx->req_q; 222 struct hisi_zip_req *q = req_q->q; 223 struct hisi_zip_req *req_cache; 224 int req_id; 225 226 write_lock(&req_q->req_lock); 227 228 req_id = find_first_zero_bit(req_q->req_bitmap, req_q->size); 229 if (req_id >= req_q->size) { 230 write_unlock(&req_q->req_lock); 231 dev_dbg(&qp_ctx->qp->qm->pdev->dev, "req cache is full!\n"); 232 return ERR_PTR(-EAGAIN); 233 } 234 set_bit(req_id, req_q->req_bitmap); 235 236 write_unlock(&req_q->req_lock); 237 238 req_cache = q + req_id; 239 req_cache->req_id = req_id; 240 req_cache->req = req; 241 242 if (is_comp) { 243 req_cache->sskip = 0; 244 req_cache->dskip = head_size; 245 } else { 246 req_cache->sskip = head_size; 247 req_cache->dskip = 0; 248 } 249 250 return req_cache; 251} 252 253static void hisi_zip_remove_req(struct hisi_zip_qp_ctx *qp_ctx, 254 struct hisi_zip_req *req) 255{ 256 struct hisi_zip_req_q *req_q = &qp_ctx->req_q; 257 258 write_lock(&req_q->req_lock); 259 clear_bit(req->req_id, req_q->req_bitmap); 260 write_unlock(&req_q->req_lock); 261} 262 263static void hisi_zip_fill_addr(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req) 264{ 265 sqe->source_addr_l = lower_32_bits(req->dma_src); 266 sqe->source_addr_h = upper_32_bits(req->dma_src); 267 sqe->dest_addr_l = lower_32_bits(req->dma_dst); 268 sqe->dest_addr_h = upper_32_bits(req->dma_dst); 269} 270 271static void hisi_zip_fill_buf_size(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req) 272{ 273 struct acomp_req *a_req = req->req; 274 275 sqe->input_data_length = a_req->slen - req->sskip; 276 sqe->dest_avail_out = a_req->dlen - req->dskip; 277 sqe->dw7 = FIELD_PREP(HZIP_IN_SGE_DATA_OFFSET_M, req->sskip); 278 sqe->dw8 = FIELD_PREP(HZIP_OUT_SGE_DATA_OFFSET_M, req->dskip); 279} 280 281static void hisi_zip_fill_buf_type(struct hisi_zip_sqe *sqe, u8 buf_type) 282{ 283 u32 val; 284 285 val = sqe->dw9 & ~HZIP_BUF_TYPE_M; 286 val |= FIELD_PREP(HZIP_BUF_TYPE_M, buf_type); 287 sqe->dw9 = val; 288} 289 290static void hisi_zip_fill_req_type(struct hisi_zip_sqe *sqe, u8 req_type) 291{ 292 u32 val; 293 294 val = sqe->dw9 & ~HZIP_REQ_TYPE_M; 295 val |= FIELD_PREP(HZIP_REQ_TYPE_M, req_type); 296 sqe->dw9 = val; 297} 298 299static void hisi_zip_fill_tag_v1(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req) 300{ 301 sqe->dw13 = req->req_id; 302} 303 304static void hisi_zip_fill_tag_v2(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req) 305{ 306 sqe->dw26 = req->req_id; 307} 308 309static void hisi_zip_fill_sqe_type(struct hisi_zip_sqe *sqe, u8 sqe_type) 310{ 311 u32 val; 312 313 val = sqe->dw7 & ~HZIP_SQE_TYPE_M; 314 val |= FIELD_PREP(HZIP_SQE_TYPE_M, sqe_type); 315 sqe->dw7 = val; 316} 317 318static void hisi_zip_fill_sqe(struct hisi_zip_ctx *ctx, struct hisi_zip_sqe *sqe, 319 u8 req_type, struct hisi_zip_req *req) 320{ 321 const struct hisi_zip_sqe_ops *ops = ctx->ops; 322 323 memset(sqe, 0, sizeof(struct hisi_zip_sqe)); 324 325 ops->fill_addr(sqe, req); 326 ops->fill_buf_size(sqe, req); 327 ops->fill_buf_type(sqe, HZIP_SGL); 328 ops->fill_req_type(sqe, req_type); 329 ops->fill_tag(sqe, req); 330 ops->fill_sqe_type(sqe, ops->sqe_type); 331} 332 333static int hisi_zip_do_work(struct hisi_zip_req *req, 334 struct hisi_zip_qp_ctx *qp_ctx) 335{ 336 struct hisi_acc_sgl_pool *pool = qp_ctx->sgl_pool; 337 struct hisi_zip_dfx *dfx = &qp_ctx->zip_dev->dfx; 338 struct acomp_req *a_req = req->req; 339 struct hisi_qp *qp = qp_ctx->qp; 340 struct device *dev = &qp->qm->pdev->dev; 341 struct hisi_zip_sqe zip_sqe; 342 int ret; 343 344 if (unlikely(!a_req->src || !a_req->slen || !a_req->dst || !a_req->dlen)) 345 return -EINVAL; 346 347 req->hw_src = hisi_acc_sg_buf_map_to_hw_sgl(dev, a_req->src, pool, 348 req->req_id << 1, &req->dma_src); 349 if (IS_ERR(req->hw_src)) { 350 dev_err(dev, "failed to map the src buffer to hw sgl (%ld)!\n", 351 PTR_ERR(req->hw_src)); 352 return PTR_ERR(req->hw_src); 353 } 354 355 req->hw_dst = hisi_acc_sg_buf_map_to_hw_sgl(dev, a_req->dst, pool, 356 (req->req_id << 1) + 1, 357 &req->dma_dst); 358 if (IS_ERR(req->hw_dst)) { 359 ret = PTR_ERR(req->hw_dst); 360 dev_err(dev, "failed to map the dst buffer to hw slg (%d)!\n", 361 ret); 362 goto err_unmap_input; 363 } 364 365 hisi_zip_fill_sqe(qp_ctx->ctx, &zip_sqe, qp->req_type, req); 366 367 /* send command to start a task */ 368 atomic64_inc(&dfx->send_cnt); 369 ret = hisi_qp_send(qp, &zip_sqe); 370 if (unlikely(ret < 0)) { 371 atomic64_inc(&dfx->send_busy_cnt); 372 ret = -EAGAIN; 373 dev_dbg_ratelimited(dev, "failed to send request!\n"); 374 goto err_unmap_output; 375 } 376 377 return -EINPROGRESS; 378 379err_unmap_output: 380 hisi_acc_sg_buf_unmap(dev, a_req->dst, req->hw_dst); 381err_unmap_input: 382 hisi_acc_sg_buf_unmap(dev, a_req->src, req->hw_src); 383 return ret; 384} 385 386static u32 hisi_zip_get_tag_v1(struct hisi_zip_sqe *sqe) 387{ 388 return sqe->dw13; 389} 390 391static u32 hisi_zip_get_tag_v2(struct hisi_zip_sqe *sqe) 392{ 393 return sqe->dw26; 394} 395 396static u32 hisi_zip_get_status(struct hisi_zip_sqe *sqe) 397{ 398 return sqe->dw3 & HZIP_BD_STATUS_M; 399} 400 401static u32 hisi_zip_get_dstlen(struct hisi_zip_sqe *sqe) 402{ 403 return sqe->produced; 404} 405 406static void hisi_zip_acomp_cb(struct hisi_qp *qp, void *data) 407{ 408 struct hisi_zip_qp_ctx *qp_ctx = qp->qp_ctx; 409 const struct hisi_zip_sqe_ops *ops = qp_ctx->ctx->ops; 410 struct hisi_zip_dfx *dfx = &qp_ctx->zip_dev->dfx; 411 struct hisi_zip_req_q *req_q = &qp_ctx->req_q; 412 struct device *dev = &qp->qm->pdev->dev; 413 struct hisi_zip_sqe *sqe = data; 414 u32 tag = ops->get_tag(sqe); 415 struct hisi_zip_req *req = req_q->q + tag; 416 struct acomp_req *acomp_req = req->req; 417 u32 status, dlen, head_size; 418 int err = 0; 419 420 atomic64_inc(&dfx->recv_cnt); 421 status = ops->get_status(sqe); 422 if (unlikely(status != 0 && status != HZIP_NC_ERR)) { 423 dev_err(dev, "%scompress fail in qp%u: %u, output: %u\n", 424 (qp->alg_type == 0) ? "" : "de", qp->qp_id, status, 425 sqe->produced); 426 atomic64_inc(&dfx->err_bd_cnt); 427 err = -EIO; 428 } 429 430 dlen = ops->get_dstlen(sqe); 431 432 hisi_acc_sg_buf_unmap(dev, acomp_req->src, req->hw_src); 433 hisi_acc_sg_buf_unmap(dev, acomp_req->dst, req->hw_dst); 434 435 head_size = (qp->alg_type == 0) ? TO_HEAD_SIZE(qp->req_type) : 0; 436 acomp_req->dlen = dlen + head_size; 437 438 if (acomp_req->base.complete) 439 acomp_request_complete(acomp_req, err); 440 441 hisi_zip_remove_req(qp_ctx, req); 442} 443 444static int hisi_zip_acompress(struct acomp_req *acomp_req) 445{ 446 struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm); 447 struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[HZIP_QPC_COMP]; 448 struct device *dev = &qp_ctx->qp->qm->pdev->dev; 449 struct hisi_zip_req *req; 450 int head_size; 451 int ret; 452 453 /* let's output compression head now */ 454 head_size = add_comp_head(acomp_req->dst, qp_ctx->qp->req_type); 455 if (unlikely(head_size < 0)) { 456 dev_err_ratelimited(dev, "failed to add comp head (%d)!\n", 457 head_size); 458 return head_size; 459 } 460 461 req = hisi_zip_create_req(acomp_req, qp_ctx, head_size, true); 462 if (IS_ERR(req)) 463 return PTR_ERR(req); 464 465 ret = hisi_zip_do_work(req, qp_ctx); 466 if (unlikely(ret != -EINPROGRESS)) { 467 dev_info_ratelimited(dev, "failed to do compress (%d)!\n", ret); 468 hisi_zip_remove_req(qp_ctx, req); 469 } 470 471 return ret; 472} 473 474static int hisi_zip_adecompress(struct acomp_req *acomp_req) 475{ 476 struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm); 477 struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[HZIP_QPC_DECOMP]; 478 struct device *dev = &qp_ctx->qp->qm->pdev->dev; 479 struct hisi_zip_req *req; 480 int head_size, ret; 481 482 head_size = get_comp_head_size(acomp_req, qp_ctx->qp->req_type); 483 if (unlikely(head_size < 0)) { 484 dev_err_ratelimited(dev, "failed to get comp head size (%d)!\n", 485 head_size); 486 return head_size; 487 } 488 489 req = hisi_zip_create_req(acomp_req, qp_ctx, head_size, false); 490 if (IS_ERR(req)) 491 return PTR_ERR(req); 492 493 ret = hisi_zip_do_work(req, qp_ctx); 494 if (unlikely(ret != -EINPROGRESS)) { 495 dev_info_ratelimited(dev, "failed to do decompress (%d)!\n", 496 ret); 497 hisi_zip_remove_req(qp_ctx, req); 498 } 499 500 return ret; 501} 502 503static int hisi_zip_start_qp(struct hisi_qp *qp, struct hisi_zip_qp_ctx *qp_ctx, 504 int alg_type, int req_type) 505{ 506 struct device *dev = &qp->qm->pdev->dev; 507 int ret; 508 509 qp->req_type = req_type; 510 qp->alg_type = alg_type; 511 qp->qp_ctx = qp_ctx; 512 513 ret = hisi_qm_start_qp(qp, 0); 514 if (ret < 0) { 515 dev_err(dev, "failed to start qp (%d)!\n", ret); 516 return ret; 517 } 518 519 qp_ctx->qp = qp; 520 521 return 0; 522} 523 524static void hisi_zip_release_qp(struct hisi_zip_qp_ctx *qp_ctx) 525{ 526 hisi_qm_stop_qp(qp_ctx->qp); 527 hisi_qm_free_qps(&qp_ctx->qp, 1); 528} 529 530static const struct hisi_zip_sqe_ops hisi_zip_ops_v1 = { 531 .sqe_type = 0, 532 .fill_addr = hisi_zip_fill_addr, 533 .fill_buf_size = hisi_zip_fill_buf_size, 534 .fill_buf_type = hisi_zip_fill_buf_type, 535 .fill_req_type = hisi_zip_fill_req_type, 536 .fill_tag = hisi_zip_fill_tag_v1, 537 .fill_sqe_type = hisi_zip_fill_sqe_type, 538 .get_tag = hisi_zip_get_tag_v1, 539 .get_status = hisi_zip_get_status, 540 .get_dstlen = hisi_zip_get_dstlen, 541}; 542 543static const struct hisi_zip_sqe_ops hisi_zip_ops_v2 = { 544 .sqe_type = 0x3, 545 .fill_addr = hisi_zip_fill_addr, 546 .fill_buf_size = hisi_zip_fill_buf_size, 547 .fill_buf_type = hisi_zip_fill_buf_type, 548 .fill_req_type = hisi_zip_fill_req_type, 549 .fill_tag = hisi_zip_fill_tag_v2, 550 .fill_sqe_type = hisi_zip_fill_sqe_type, 551 .get_tag = hisi_zip_get_tag_v2, 552 .get_status = hisi_zip_get_status, 553 .get_dstlen = hisi_zip_get_dstlen, 554}; 555 556static int hisi_zip_ctx_init(struct hisi_zip_ctx *hisi_zip_ctx, u8 req_type, int node) 557{ 558 struct hisi_qp *qps[HZIP_CTX_Q_NUM] = { NULL }; 559 struct hisi_zip_qp_ctx *qp_ctx; 560 struct hisi_zip *hisi_zip; 561 int ret, i, j; 562 563 ret = zip_create_qps(qps, HZIP_CTX_Q_NUM, node); 564 if (ret) { 565 pr_err("failed to create zip qps (%d)!\n", ret); 566 return -ENODEV; 567 } 568 569 hisi_zip = container_of(qps[0]->qm, struct hisi_zip, qm); 570 571 for (i = 0; i < HZIP_CTX_Q_NUM; i++) { 572 /* alg_type = 0 for compress, 1 for decompress in hw sqe */ 573 qp_ctx = &hisi_zip_ctx->qp_ctx[i]; 574 qp_ctx->ctx = hisi_zip_ctx; 575 ret = hisi_zip_start_qp(qps[i], qp_ctx, i, req_type); 576 if (ret) { 577 for (j = i - 1; j >= 0; j--) 578 hisi_qm_stop_qp(hisi_zip_ctx->qp_ctx[j].qp); 579 580 hisi_qm_free_qps(qps, HZIP_CTX_Q_NUM); 581 return ret; 582 } 583 584 qp_ctx->zip_dev = hisi_zip; 585 } 586 587 if (hisi_zip->qm.ver < QM_HW_V3) 588 hisi_zip_ctx->ops = &hisi_zip_ops_v1; 589 else 590 hisi_zip_ctx->ops = &hisi_zip_ops_v2; 591 592 return 0; 593} 594 595static void hisi_zip_ctx_exit(struct hisi_zip_ctx *hisi_zip_ctx) 596{ 597 int i; 598 599 for (i = 0; i < HZIP_CTX_Q_NUM; i++) 600 hisi_zip_release_qp(&hisi_zip_ctx->qp_ctx[i]); 601} 602 603static int hisi_zip_create_req_q(struct hisi_zip_ctx *ctx) 604{ 605 u16 q_depth = ctx->qp_ctx[0].qp->sq_depth; 606 struct hisi_zip_req_q *req_q; 607 int i, ret; 608 609 for (i = 0; i < HZIP_CTX_Q_NUM; i++) { 610 req_q = &ctx->qp_ctx[i].req_q; 611 req_q->size = q_depth; 612 613 req_q->req_bitmap = bitmap_zalloc(req_q->size, GFP_KERNEL); 614 if (!req_q->req_bitmap) { 615 ret = -ENOMEM; 616 if (i == 0) 617 return ret; 618 619 goto err_free_comp_q; 620 } 621 rwlock_init(&req_q->req_lock); 622 623 req_q->q = kcalloc(req_q->size, sizeof(struct hisi_zip_req), 624 GFP_KERNEL); 625 if (!req_q->q) { 626 ret = -ENOMEM; 627 if (i == 0) 628 goto err_free_comp_bitmap; 629 else 630 goto err_free_decomp_bitmap; 631 } 632 } 633 634 return 0; 635 636err_free_decomp_bitmap: 637 bitmap_free(ctx->qp_ctx[HZIP_QPC_DECOMP].req_q.req_bitmap); 638err_free_comp_q: 639 kfree(ctx->qp_ctx[HZIP_QPC_COMP].req_q.q); 640err_free_comp_bitmap: 641 bitmap_free(ctx->qp_ctx[HZIP_QPC_COMP].req_q.req_bitmap); 642 return ret; 643} 644 645static void hisi_zip_release_req_q(struct hisi_zip_ctx *ctx) 646{ 647 int i; 648 649 for (i = 0; i < HZIP_CTX_Q_NUM; i++) { 650 kfree(ctx->qp_ctx[i].req_q.q); 651 bitmap_free(ctx->qp_ctx[i].req_q.req_bitmap); 652 } 653} 654 655static int hisi_zip_create_sgl_pool(struct hisi_zip_ctx *ctx) 656{ 657 u16 q_depth = ctx->qp_ctx[0].qp->sq_depth; 658 struct hisi_zip_qp_ctx *tmp; 659 struct device *dev; 660 int i; 661 662 for (i = 0; i < HZIP_CTX_Q_NUM; i++) { 663 tmp = &ctx->qp_ctx[i]; 664 dev = &tmp->qp->qm->pdev->dev; 665 tmp->sgl_pool = hisi_acc_create_sgl_pool(dev, q_depth << 1, 666 sgl_sge_nr); 667 if (IS_ERR(tmp->sgl_pool)) { 668 if (i == 1) 669 goto err_free_sgl_pool0; 670 return -ENOMEM; 671 } 672 } 673 674 return 0; 675 676err_free_sgl_pool0: 677 hisi_acc_free_sgl_pool(&ctx->qp_ctx[HZIP_QPC_COMP].qp->qm->pdev->dev, 678 ctx->qp_ctx[HZIP_QPC_COMP].sgl_pool); 679 return -ENOMEM; 680} 681 682static void hisi_zip_release_sgl_pool(struct hisi_zip_ctx *ctx) 683{ 684 int i; 685 686 for (i = 0; i < HZIP_CTX_Q_NUM; i++) 687 hisi_acc_free_sgl_pool(&ctx->qp_ctx[i].qp->qm->pdev->dev, 688 ctx->qp_ctx[i].sgl_pool); 689} 690 691static void hisi_zip_set_acomp_cb(struct hisi_zip_ctx *ctx, 692 void (*fn)(struct hisi_qp *, void *)) 693{ 694 int i; 695 696 for (i = 0; i < HZIP_CTX_Q_NUM; i++) 697 ctx->qp_ctx[i].qp->req_cb = fn; 698} 699 700static int hisi_zip_acomp_init(struct crypto_acomp *tfm) 701{ 702 const char *alg_name = crypto_tfm_alg_name(&tfm->base); 703 struct hisi_zip_ctx *ctx = crypto_tfm_ctx(&tfm->base); 704 struct device *dev; 705 int ret; 706 707 ret = hisi_zip_ctx_init(ctx, COMP_NAME_TO_TYPE(alg_name), tfm->base.node); 708 if (ret) { 709 pr_err("failed to init ctx (%d)!\n", ret); 710 return ret; 711 } 712 713 dev = &ctx->qp_ctx[0].qp->qm->pdev->dev; 714 715 ret = hisi_zip_create_req_q(ctx); 716 if (ret) { 717 dev_err(dev, "failed to create request queue (%d)!\n", ret); 718 goto err_ctx_exit; 719 } 720 721 ret = hisi_zip_create_sgl_pool(ctx); 722 if (ret) { 723 dev_err(dev, "failed to create sgl pool (%d)!\n", ret); 724 goto err_release_req_q; 725 } 726 727 hisi_zip_set_acomp_cb(ctx, hisi_zip_acomp_cb); 728 729 return 0; 730 731err_release_req_q: 732 hisi_zip_release_req_q(ctx); 733err_ctx_exit: 734 hisi_zip_ctx_exit(ctx); 735 return ret; 736} 737 738static void hisi_zip_acomp_exit(struct crypto_acomp *tfm) 739{ 740 struct hisi_zip_ctx *ctx = crypto_tfm_ctx(&tfm->base); 741 742 hisi_zip_set_acomp_cb(ctx, NULL); 743 hisi_zip_release_sgl_pool(ctx); 744 hisi_zip_release_req_q(ctx); 745 hisi_zip_ctx_exit(ctx); 746} 747 748static struct acomp_alg hisi_zip_acomp_zlib = { 749 .init = hisi_zip_acomp_init, 750 .exit = hisi_zip_acomp_exit, 751 .compress = hisi_zip_acompress, 752 .decompress = hisi_zip_adecompress, 753 .base = { 754 .cra_name = "zlib-deflate", 755 .cra_driver_name = "hisi-zlib-acomp", 756 .cra_module = THIS_MODULE, 757 .cra_priority = HZIP_ALG_PRIORITY, 758 .cra_ctxsize = sizeof(struct hisi_zip_ctx), 759 } 760}; 761 762static int hisi_zip_register_zlib(struct hisi_qm *qm) 763{ 764 int ret; 765 766 if (!hisi_zip_alg_support(qm, HZIP_ALG_ZLIB)) 767 return 0; 768 769 ret = crypto_register_acomp(&hisi_zip_acomp_zlib); 770 if (ret) 771 dev_err(&qm->pdev->dev, "failed to register to zlib (%d)!\n", ret); 772 773 return ret; 774} 775 776static void hisi_zip_unregister_zlib(struct hisi_qm *qm) 777{ 778 if (!hisi_zip_alg_support(qm, HZIP_ALG_ZLIB)) 779 return; 780 781 crypto_unregister_acomp(&hisi_zip_acomp_zlib); 782} 783 784static struct acomp_alg hisi_zip_acomp_gzip = { 785 .init = hisi_zip_acomp_init, 786 .exit = hisi_zip_acomp_exit, 787 .compress = hisi_zip_acompress, 788 .decompress = hisi_zip_adecompress, 789 .base = { 790 .cra_name = "gzip", 791 .cra_driver_name = "hisi-gzip-acomp", 792 .cra_module = THIS_MODULE, 793 .cra_priority = HZIP_ALG_PRIORITY, 794 .cra_ctxsize = sizeof(struct hisi_zip_ctx), 795 } 796}; 797 798static int hisi_zip_register_gzip(struct hisi_qm *qm) 799{ 800 int ret; 801 802 if (!hisi_zip_alg_support(qm, HZIP_ALG_GZIP)) 803 return 0; 804 805 ret = crypto_register_acomp(&hisi_zip_acomp_gzip); 806 if (ret) 807 dev_err(&qm->pdev->dev, "failed to register to gzip (%d)!\n", ret); 808 809 return ret; 810} 811 812static void hisi_zip_unregister_gzip(struct hisi_qm *qm) 813{ 814 if (!hisi_zip_alg_support(qm, HZIP_ALG_GZIP)) 815 return; 816 817 crypto_unregister_acomp(&hisi_zip_acomp_gzip); 818} 819 820int hisi_zip_register_to_crypto(struct hisi_qm *qm) 821{ 822 int ret = 0; 823 824 ret = hisi_zip_register_zlib(qm); 825 if (ret) 826 return ret; 827 828 ret = hisi_zip_register_gzip(qm); 829 if (ret) 830 hisi_zip_unregister_zlib(qm); 831 832 return ret; 833} 834 835void hisi_zip_unregister_from_crypto(struct hisi_qm *qm) 836{ 837 hisi_zip_unregister_zlib(qm); 838 hisi_zip_unregister_gzip(qm); 839} 840