1// SPDX-License-Identifier: GPL-2.0-or-later 2/* 3 * Driver for IBM Power 842 compression accelerator 4 * 5 * Copyright (C) IBM Corporation, 2012 6 * 7 * Authors: Robert Jennings <rcj@linux.vnet.ibm.com> 8 * Seth Jennings <sjenning@linux.vnet.ibm.com> 9 */ 10 11#include <asm/vio.h> 12#include <asm/hvcall.h> 13#include <asm/vas.h> 14 15#include "nx-842.h" 16#include "nx_csbcpb.h" /* struct nx_csbcpb */ 17 18MODULE_LICENSE("GPL"); 19MODULE_AUTHOR("Robert Jennings <rcj@linux.vnet.ibm.com>"); 20MODULE_DESCRIPTION("842 H/W Compression driver for IBM Power processors"); 21MODULE_ALIAS_CRYPTO("842"); 22MODULE_ALIAS_CRYPTO("842-nx"); 23 24/* 25 * Coprocessor type specific capabilities from the hypervisor. 26 */ 27struct hv_nx_cop_caps { 28 __be64 descriptor; 29 __be64 req_max_processed_len; /* Max bytes in one GZIP request */ 30 __be64 min_compress_len; /* Min compression size in bytes */ 31 __be64 min_decompress_len; /* Min decompression size in bytes */ 32} __packed __aligned(0x1000); 33 34/* 35 * Coprocessor type specific capabilities. 36 */ 37struct nx_cop_caps { 38 u64 descriptor; 39 u64 req_max_processed_len; /* Max bytes in one GZIP request */ 40 u64 min_compress_len; /* Min compression in bytes */ 41 u64 min_decompress_len; /* Min decompression in bytes */ 42}; 43 44static u64 caps_feat; 45static struct nx_cop_caps nx_cop_caps; 46 47static struct nx842_constraints nx842_pseries_constraints = { 48 .alignment = DDE_BUFFER_ALIGN, 49 .multiple = DDE_BUFFER_LAST_MULT, 50 .minimum = DDE_BUFFER_LAST_MULT, 51 .maximum = PAGE_SIZE, /* dynamic, max_sync_size */ 52}; 53 54static int check_constraints(unsigned long buf, unsigned int *len, bool in) 55{ 56 if (!IS_ALIGNED(buf, nx842_pseries_constraints.alignment)) { 57 pr_debug("%s buffer 0x%lx not aligned to 0x%x\n", 58 in ? "input" : "output", buf, 59 nx842_pseries_constraints.alignment); 60 return -EINVAL; 61 } 62 if (*len % nx842_pseries_constraints.multiple) { 63 pr_debug("%s buffer len 0x%x not multiple of 0x%x\n", 64 in ? "input" : "output", *len, 65 nx842_pseries_constraints.multiple); 66 if (in) 67 return -EINVAL; 68 *len = round_down(*len, nx842_pseries_constraints.multiple); 69 } 70 if (*len < nx842_pseries_constraints.minimum) { 71 pr_debug("%s buffer len 0x%x under minimum 0x%x\n", 72 in ? "input" : "output", *len, 73 nx842_pseries_constraints.minimum); 74 return -EINVAL; 75 } 76 if (*len > nx842_pseries_constraints.maximum) { 77 pr_debug("%s buffer len 0x%x over maximum 0x%x\n", 78 in ? "input" : "output", *len, 79 nx842_pseries_constraints.maximum); 80 if (in) 81 return -EINVAL; 82 *len = nx842_pseries_constraints.maximum; 83 } 84 return 0; 85} 86 87/* I assume we need to align the CSB? */ 88#define WORKMEM_ALIGN (256) 89 90struct nx842_workmem { 91 /* scatterlist */ 92 char slin[4096]; 93 char slout[4096]; 94 /* coprocessor status/parameter block */ 95 struct nx_csbcpb csbcpb; 96 97 char padding[WORKMEM_ALIGN]; 98} __aligned(WORKMEM_ALIGN); 99 100/* Macros for fields within nx_csbcpb */ 101/* Check the valid bit within the csbcpb valid field */ 102#define NX842_CSBCBP_VALID_CHK(x) (x & BIT_MASK(7)) 103 104/* CE macros operate on the completion_extension field bits in the csbcpb. 105 * CE0 0=full completion, 1=partial completion 106 * CE1 0=CE0 indicates completion, 1=termination (output may be modified) 107 * CE2 0=processed_bytes is source bytes, 1=processed_bytes is target bytes */ 108#define NX842_CSBCPB_CE0(x) (x & BIT_MASK(7)) 109#define NX842_CSBCPB_CE1(x) (x & BIT_MASK(6)) 110#define NX842_CSBCPB_CE2(x) (x & BIT_MASK(5)) 111 112/* The NX unit accepts data only on 4K page boundaries */ 113#define NX842_HW_PAGE_SIZE (4096) 114#define NX842_HW_PAGE_MASK (~(NX842_HW_PAGE_SIZE-1)) 115 116struct ibm_nx842_counters { 117 atomic64_t comp_complete; 118 atomic64_t comp_failed; 119 atomic64_t decomp_complete; 120 atomic64_t decomp_failed; 121 atomic64_t swdecomp; 122 atomic64_t comp_times[32]; 123 atomic64_t decomp_times[32]; 124}; 125 126struct nx842_devdata { 127 struct vio_dev *vdev; 128 struct device *dev; 129 struct ibm_nx842_counters *counters; 130 unsigned int max_sg_len; 131 unsigned int max_sync_size; 132 unsigned int max_sync_sg; 133}; 134 135static struct nx842_devdata __rcu *devdata; 136static DEFINE_SPINLOCK(devdata_mutex); 137 138#define NX842_COUNTER_INC(_x) \ 139static inline void nx842_inc_##_x( \ 140 const struct nx842_devdata *dev) { \ 141 if (dev) \ 142 atomic64_inc(&dev->counters->_x); \ 143} 144NX842_COUNTER_INC(comp_complete); 145NX842_COUNTER_INC(comp_failed); 146NX842_COUNTER_INC(decomp_complete); 147NX842_COUNTER_INC(decomp_failed); 148NX842_COUNTER_INC(swdecomp); 149 150#define NX842_HIST_SLOTS 16 151 152static void ibm_nx842_incr_hist(atomic64_t *times, unsigned int time) 153{ 154 int bucket = fls(time); 155 156 if (bucket) 157 bucket = min((NX842_HIST_SLOTS - 1), bucket - 1); 158 159 atomic64_inc(×[bucket]); 160} 161 162/* NX unit operation flags */ 163#define NX842_OP_COMPRESS 0x0 164#define NX842_OP_CRC 0x1 165#define NX842_OP_DECOMPRESS 0x2 166#define NX842_OP_COMPRESS_CRC (NX842_OP_COMPRESS | NX842_OP_CRC) 167#define NX842_OP_DECOMPRESS_CRC (NX842_OP_DECOMPRESS | NX842_OP_CRC) 168#define NX842_OP_ASYNC (1<<23) 169#define NX842_OP_NOTIFY (1<<22) 170#define NX842_OP_NOTIFY_INT(x) ((x & 0xff)<<8) 171 172static unsigned long nx842_get_desired_dma(struct vio_dev *viodev) 173{ 174 /* No use of DMA mappings within the driver. */ 175 return 0; 176} 177 178struct nx842_slentry { 179 __be64 ptr; /* Real address (use __pa()) */ 180 __be64 len; 181}; 182 183/* pHyp scatterlist entry */ 184struct nx842_scatterlist { 185 int entry_nr; /* number of slentries */ 186 struct nx842_slentry *entries; /* ptr to array of slentries */ 187}; 188 189/* Does not include sizeof(entry_nr) in the size */ 190static inline unsigned long nx842_get_scatterlist_size( 191 struct nx842_scatterlist *sl) 192{ 193 return sl->entry_nr * sizeof(struct nx842_slentry); 194} 195 196static int nx842_build_scatterlist(unsigned long buf, int len, 197 struct nx842_scatterlist *sl) 198{ 199 unsigned long entrylen; 200 struct nx842_slentry *entry; 201 202 sl->entry_nr = 0; 203 204 entry = sl->entries; 205 while (len) { 206 entry->ptr = cpu_to_be64(nx842_get_pa((void *)buf)); 207 entrylen = min_t(int, len, 208 LEN_ON_SIZE(buf, NX842_HW_PAGE_SIZE)); 209 entry->len = cpu_to_be64(entrylen); 210 211 len -= entrylen; 212 buf += entrylen; 213 214 sl->entry_nr++; 215 entry++; 216 } 217 218 return 0; 219} 220 221static int nx842_validate_result(struct device *dev, 222 struct cop_status_block *csb) 223{ 224 /* The csb must be valid after returning from vio_h_cop_sync */ 225 if (!NX842_CSBCBP_VALID_CHK(csb->valid)) { 226 dev_err(dev, "%s: cspcbp not valid upon completion.\n", 227 __func__); 228 dev_dbg(dev, "valid:0x%02x cs:0x%02x cc:0x%02x ce:0x%02x\n", 229 csb->valid, 230 csb->crb_seq_number, 231 csb->completion_code, 232 csb->completion_extension); 233 dev_dbg(dev, "processed_bytes:%d address:0x%016lx\n", 234 be32_to_cpu(csb->processed_byte_count), 235 (unsigned long)be64_to_cpu(csb->address)); 236 return -EIO; 237 } 238 239 /* Check return values from the hardware in the CSB */ 240 switch (csb->completion_code) { 241 case 0: /* Completed without error */ 242 break; 243 case 64: /* Compression ok, but output larger than input */ 244 dev_dbg(dev, "%s: output size larger than input size\n", 245 __func__); 246 break; 247 case 13: /* Output buffer too small */ 248 dev_dbg(dev, "%s: Out of space in output buffer\n", 249 __func__); 250 return -ENOSPC; 251 case 65: /* Calculated CRC doesn't match the passed value */ 252 dev_dbg(dev, "%s: CRC mismatch for decompression\n", 253 __func__); 254 return -EINVAL; 255 case 66: /* Input data contains an illegal template field */ 256 case 67: /* Template indicates data past the end of the input stream */ 257 dev_dbg(dev, "%s: Bad data for decompression (code:%d)\n", 258 __func__, csb->completion_code); 259 return -EINVAL; 260 default: 261 dev_dbg(dev, "%s: Unspecified error (code:%d)\n", 262 __func__, csb->completion_code); 263 return -EIO; 264 } 265 266 /* Hardware sanity check */ 267 if (!NX842_CSBCPB_CE2(csb->completion_extension)) { 268 dev_err(dev, "%s: No error returned by hardware, but " 269 "data returned is unusable, contact support.\n" 270 "(Additional info: csbcbp->processed bytes " 271 "does not specify processed bytes for the " 272 "target buffer.)\n", __func__); 273 return -EIO; 274 } 275 276 return 0; 277} 278 279/** 280 * nx842_pseries_compress - Compress data using the 842 algorithm 281 * 282 * Compression provide by the NX842 coprocessor on IBM Power systems. 283 * The input buffer is compressed and the result is stored in the 284 * provided output buffer. 285 * 286 * Upon return from this function @outlen contains the length of the 287 * compressed data. If there is an error then @outlen will be 0 and an 288 * error will be specified by the return code from this function. 289 * 290 * @in: Pointer to input buffer 291 * @inlen: Length of input buffer 292 * @out: Pointer to output buffer 293 * @outlen: Length of output buffer 294 * @wmem: ptr to buffer for working memory, size determined by 295 * nx842_pseries_driver.workmem_size 296 * 297 * Returns: 298 * 0 Success, output of length @outlen stored in the buffer at @out 299 * -ENOMEM Unable to allocate internal buffers 300 * -ENOSPC Output buffer is to small 301 * -EIO Internal error 302 * -ENODEV Hardware unavailable 303 */ 304static int nx842_pseries_compress(const unsigned char *in, unsigned int inlen, 305 unsigned char *out, unsigned int *outlen, 306 void *wmem) 307{ 308 struct nx842_devdata *local_devdata; 309 struct device *dev = NULL; 310 struct nx842_workmem *workmem; 311 struct nx842_scatterlist slin, slout; 312 struct nx_csbcpb *csbcpb; 313 int ret = 0; 314 unsigned long inbuf, outbuf; 315 struct vio_pfo_op op = { 316 .done = NULL, 317 .handle = 0, 318 .timeout = 0, 319 }; 320 unsigned long start = get_tb(); 321 322 inbuf = (unsigned long)in; 323 if (check_constraints(inbuf, &inlen, true)) 324 return -EINVAL; 325 326 outbuf = (unsigned long)out; 327 if (check_constraints(outbuf, outlen, false)) 328 return -EINVAL; 329 330 rcu_read_lock(); 331 local_devdata = rcu_dereference(devdata); 332 if (!local_devdata || !local_devdata->dev) { 333 rcu_read_unlock(); 334 return -ENODEV; 335 } 336 dev = local_devdata->dev; 337 338 /* Init scatterlist */ 339 workmem = PTR_ALIGN(wmem, WORKMEM_ALIGN); 340 slin.entries = (struct nx842_slentry *)workmem->slin; 341 slout.entries = (struct nx842_slentry *)workmem->slout; 342 343 /* Init operation */ 344 op.flags = NX842_OP_COMPRESS_CRC; 345 csbcpb = &workmem->csbcpb; 346 memset(csbcpb, 0, sizeof(*csbcpb)); 347 op.csbcpb = nx842_get_pa(csbcpb); 348 349 if ((inbuf & NX842_HW_PAGE_MASK) == 350 ((inbuf + inlen - 1) & NX842_HW_PAGE_MASK)) { 351 /* Create direct DDE */ 352 op.in = nx842_get_pa((void *)inbuf); 353 op.inlen = inlen; 354 } else { 355 /* Create indirect DDE (scatterlist) */ 356 nx842_build_scatterlist(inbuf, inlen, &slin); 357 op.in = nx842_get_pa(slin.entries); 358 op.inlen = -nx842_get_scatterlist_size(&slin); 359 } 360 361 if ((outbuf & NX842_HW_PAGE_MASK) == 362 ((outbuf + *outlen - 1) & NX842_HW_PAGE_MASK)) { 363 /* Create direct DDE */ 364 op.out = nx842_get_pa((void *)outbuf); 365 op.outlen = *outlen; 366 } else { 367 /* Create indirect DDE (scatterlist) */ 368 nx842_build_scatterlist(outbuf, *outlen, &slout); 369 op.out = nx842_get_pa(slout.entries); 370 op.outlen = -nx842_get_scatterlist_size(&slout); 371 } 372 373 dev_dbg(dev, "%s: op.in %lx op.inlen %ld op.out %lx op.outlen %ld\n", 374 __func__, (unsigned long)op.in, (long)op.inlen, 375 (unsigned long)op.out, (long)op.outlen); 376 377 /* Send request to pHyp */ 378 ret = vio_h_cop_sync(local_devdata->vdev, &op); 379 380 /* Check for pHyp error */ 381 if (ret) { 382 dev_dbg(dev, "%s: vio_h_cop_sync error (ret=%d, hret=%ld)\n", 383 __func__, ret, op.hcall_err); 384 ret = -EIO; 385 goto unlock; 386 } 387 388 /* Check for hardware error */ 389 ret = nx842_validate_result(dev, &csbcpb->csb); 390 if (ret) 391 goto unlock; 392 393 *outlen = be32_to_cpu(csbcpb->csb.processed_byte_count); 394 dev_dbg(dev, "%s: processed_bytes=%d\n", __func__, *outlen); 395 396unlock: 397 if (ret) 398 nx842_inc_comp_failed(local_devdata); 399 else { 400 nx842_inc_comp_complete(local_devdata); 401 ibm_nx842_incr_hist(local_devdata->counters->comp_times, 402 (get_tb() - start) / tb_ticks_per_usec); 403 } 404 rcu_read_unlock(); 405 return ret; 406} 407 408/** 409 * nx842_pseries_decompress - Decompress data using the 842 algorithm 410 * 411 * Decompression provide by the NX842 coprocessor on IBM Power systems. 412 * The input buffer is decompressed and the result is stored in the 413 * provided output buffer. The size allocated to the output buffer is 414 * provided by the caller of this function in @outlen. Upon return from 415 * this function @outlen contains the length of the decompressed data. 416 * If there is an error then @outlen will be 0 and an error will be 417 * specified by the return code from this function. 418 * 419 * @in: Pointer to input buffer 420 * @inlen: Length of input buffer 421 * @out: Pointer to output buffer 422 * @outlen: Length of output buffer 423 * @wmem: ptr to buffer for working memory, size determined by 424 * nx842_pseries_driver.workmem_size 425 * 426 * Returns: 427 * 0 Success, output of length @outlen stored in the buffer at @out 428 * -ENODEV Hardware decompression device is unavailable 429 * -ENOMEM Unable to allocate internal buffers 430 * -ENOSPC Output buffer is to small 431 * -EINVAL Bad input data encountered when attempting decompress 432 * -EIO Internal error 433 */ 434static int nx842_pseries_decompress(const unsigned char *in, unsigned int inlen, 435 unsigned char *out, unsigned int *outlen, 436 void *wmem) 437{ 438 struct nx842_devdata *local_devdata; 439 struct device *dev = NULL; 440 struct nx842_workmem *workmem; 441 struct nx842_scatterlist slin, slout; 442 struct nx_csbcpb *csbcpb; 443 int ret = 0; 444 unsigned long inbuf, outbuf; 445 struct vio_pfo_op op = { 446 .done = NULL, 447 .handle = 0, 448 .timeout = 0, 449 }; 450 unsigned long start = get_tb(); 451 452 /* Ensure page alignment and size */ 453 inbuf = (unsigned long)in; 454 if (check_constraints(inbuf, &inlen, true)) 455 return -EINVAL; 456 457 outbuf = (unsigned long)out; 458 if (check_constraints(outbuf, outlen, false)) 459 return -EINVAL; 460 461 rcu_read_lock(); 462 local_devdata = rcu_dereference(devdata); 463 if (!local_devdata || !local_devdata->dev) { 464 rcu_read_unlock(); 465 return -ENODEV; 466 } 467 dev = local_devdata->dev; 468 469 workmem = PTR_ALIGN(wmem, WORKMEM_ALIGN); 470 471 /* Init scatterlist */ 472 slin.entries = (struct nx842_slentry *)workmem->slin; 473 slout.entries = (struct nx842_slentry *)workmem->slout; 474 475 /* Init operation */ 476 op.flags = NX842_OP_DECOMPRESS_CRC; 477 csbcpb = &workmem->csbcpb; 478 memset(csbcpb, 0, sizeof(*csbcpb)); 479 op.csbcpb = nx842_get_pa(csbcpb); 480 481 if ((inbuf & NX842_HW_PAGE_MASK) == 482 ((inbuf + inlen - 1) & NX842_HW_PAGE_MASK)) { 483 /* Create direct DDE */ 484 op.in = nx842_get_pa((void *)inbuf); 485 op.inlen = inlen; 486 } else { 487 /* Create indirect DDE (scatterlist) */ 488 nx842_build_scatterlist(inbuf, inlen, &slin); 489 op.in = nx842_get_pa(slin.entries); 490 op.inlen = -nx842_get_scatterlist_size(&slin); 491 } 492 493 if ((outbuf & NX842_HW_PAGE_MASK) == 494 ((outbuf + *outlen - 1) & NX842_HW_PAGE_MASK)) { 495 /* Create direct DDE */ 496 op.out = nx842_get_pa((void *)outbuf); 497 op.outlen = *outlen; 498 } else { 499 /* Create indirect DDE (scatterlist) */ 500 nx842_build_scatterlist(outbuf, *outlen, &slout); 501 op.out = nx842_get_pa(slout.entries); 502 op.outlen = -nx842_get_scatterlist_size(&slout); 503 } 504 505 dev_dbg(dev, "%s: op.in %lx op.inlen %ld op.out %lx op.outlen %ld\n", 506 __func__, (unsigned long)op.in, (long)op.inlen, 507 (unsigned long)op.out, (long)op.outlen); 508 509 /* Send request to pHyp */ 510 ret = vio_h_cop_sync(local_devdata->vdev, &op); 511 512 /* Check for pHyp error */ 513 if (ret) { 514 dev_dbg(dev, "%s: vio_h_cop_sync error (ret=%d, hret=%ld)\n", 515 __func__, ret, op.hcall_err); 516 goto unlock; 517 } 518 519 /* Check for hardware error */ 520 ret = nx842_validate_result(dev, &csbcpb->csb); 521 if (ret) 522 goto unlock; 523 524 *outlen = be32_to_cpu(csbcpb->csb.processed_byte_count); 525 526unlock: 527 if (ret) 528 /* decompress fail */ 529 nx842_inc_decomp_failed(local_devdata); 530 else { 531 nx842_inc_decomp_complete(local_devdata); 532 ibm_nx842_incr_hist(local_devdata->counters->decomp_times, 533 (get_tb() - start) / tb_ticks_per_usec); 534 } 535 536 rcu_read_unlock(); 537 return ret; 538} 539 540/** 541 * nx842_OF_set_defaults -- Set default (disabled) values for devdata 542 * 543 * @devdata: struct nx842_devdata to update 544 * 545 * Returns: 546 * 0 on success 547 * -ENOENT if @devdata ptr is NULL 548 */ 549static int nx842_OF_set_defaults(struct nx842_devdata *devdata) 550{ 551 if (devdata) { 552 devdata->max_sync_size = 0; 553 devdata->max_sync_sg = 0; 554 devdata->max_sg_len = 0; 555 return 0; 556 } else 557 return -ENOENT; 558} 559 560/** 561 * nx842_OF_upd_status -- Check the device info from OF status prop 562 * 563 * The status property indicates if the accelerator is enabled. If the 564 * device is in the OF tree it indicates that the hardware is present. 565 * The status field indicates if the device is enabled when the status 566 * is 'okay'. Otherwise the device driver will be disabled. 567 * 568 * @devdata: struct nx842_devdata to use for dev_info 569 * @prop: struct property point containing the maxsyncop for the update 570 * 571 * Returns: 572 * 0 - Device is available 573 * -ENODEV - Device is not available 574 */ 575static int nx842_OF_upd_status(struct nx842_devdata *devdata, 576 struct property *prop) 577{ 578 const char *status = (const char *)prop->value; 579 580 if (!strncmp(status, "okay", (size_t)prop->length)) 581 return 0; 582 if (!strncmp(status, "disabled", (size_t)prop->length)) 583 return -ENODEV; 584 dev_info(devdata->dev, "%s: unknown status '%s'\n", __func__, status); 585 586 return -EINVAL; 587} 588 589/** 590 * nx842_OF_upd_maxsglen -- Update the device info from OF maxsglen prop 591 * 592 * Definition of the 'ibm,max-sg-len' OF property: 593 * This field indicates the maximum byte length of a scatter list 594 * for the platform facility. It is a single cell encoded as with encode-int. 595 * 596 * Example: 597 * # od -x ibm,max-sg-len 598 * 0000000 0000 0ff0 599 * 600 * In this example, the maximum byte length of a scatter list is 601 * 0x0ff0 (4,080). 602 * 603 * @devdata: struct nx842_devdata to update 604 * @prop: struct property point containing the maxsyncop for the update 605 * 606 * Returns: 607 * 0 on success 608 * -EINVAL on failure 609 */ 610static int nx842_OF_upd_maxsglen(struct nx842_devdata *devdata, 611 struct property *prop) { 612 int ret = 0; 613 const unsigned int maxsglen = of_read_number(prop->value, 1); 614 615 if (prop->length != sizeof(maxsglen)) { 616 dev_err(devdata->dev, "%s: unexpected format for ibm,max-sg-len property\n", __func__); 617 dev_dbg(devdata->dev, "%s: ibm,max-sg-len is %d bytes long, expected %lu bytes\n", __func__, 618 prop->length, sizeof(maxsglen)); 619 ret = -EINVAL; 620 } else { 621 devdata->max_sg_len = min_t(unsigned int, 622 maxsglen, NX842_HW_PAGE_SIZE); 623 } 624 625 return ret; 626} 627 628/** 629 * nx842_OF_upd_maxsyncop -- Update the device info from OF maxsyncop prop 630 * 631 * Definition of the 'ibm,max-sync-cop' OF property: 632 * Two series of cells. The first series of cells represents the maximums 633 * that can be synchronously compressed. The second series of cells 634 * represents the maximums that can be synchronously decompressed. 635 * 1. The first cell in each series contains the count of the number of 636 * data length, scatter list elements pairs that follow – each being 637 * of the form 638 * a. One cell data byte length 639 * b. One cell total number of scatter list elements 640 * 641 * Example: 642 * # od -x ibm,max-sync-cop 643 * 0000000 0000 0001 0000 1000 0000 01fe 0000 0001 644 * 0000020 0000 1000 0000 01fe 645 * 646 * In this example, compression supports 0x1000 (4,096) data byte length 647 * and 0x1fe (510) total scatter list elements. Decompression supports 648 * 0x1000 (4,096) data byte length and 0x1f3 (510) total scatter list 649 * elements. 650 * 651 * @devdata: struct nx842_devdata to update 652 * @prop: struct property point containing the maxsyncop for the update 653 * 654 * Returns: 655 * 0 on success 656 * -EINVAL on failure 657 */ 658static int nx842_OF_upd_maxsyncop(struct nx842_devdata *devdata, 659 struct property *prop) { 660 int ret = 0; 661 unsigned int comp_data_limit, decomp_data_limit; 662 unsigned int comp_sg_limit, decomp_sg_limit; 663 const struct maxsynccop_t { 664 __be32 comp_elements; 665 __be32 comp_data_limit; 666 __be32 comp_sg_limit; 667 __be32 decomp_elements; 668 __be32 decomp_data_limit; 669 __be32 decomp_sg_limit; 670 } *maxsynccop; 671 672 if (prop->length != sizeof(*maxsynccop)) { 673 dev_err(devdata->dev, "%s: unexpected format for ibm,max-sync-cop property\n", __func__); 674 dev_dbg(devdata->dev, "%s: ibm,max-sync-cop is %d bytes long, expected %lu bytes\n", __func__, prop->length, 675 sizeof(*maxsynccop)); 676 ret = -EINVAL; 677 goto out; 678 } 679 680 maxsynccop = (const struct maxsynccop_t *)prop->value; 681 comp_data_limit = be32_to_cpu(maxsynccop->comp_data_limit); 682 comp_sg_limit = be32_to_cpu(maxsynccop->comp_sg_limit); 683 decomp_data_limit = be32_to_cpu(maxsynccop->decomp_data_limit); 684 decomp_sg_limit = be32_to_cpu(maxsynccop->decomp_sg_limit); 685 686 /* Use one limit rather than separate limits for compression and 687 * decompression. Set a maximum for this so as not to exceed the 688 * size that the header can support and round the value down to 689 * the hardware page size (4K) */ 690 devdata->max_sync_size = min(comp_data_limit, decomp_data_limit); 691 692 devdata->max_sync_size = min_t(unsigned int, devdata->max_sync_size, 693 65536); 694 695 if (devdata->max_sync_size < 4096) { 696 dev_err(devdata->dev, "%s: hardware max data size (%u) is " 697 "less than the driver minimum, unable to use " 698 "the hardware device\n", 699 __func__, devdata->max_sync_size); 700 ret = -EINVAL; 701 goto out; 702 } 703 704 nx842_pseries_constraints.maximum = devdata->max_sync_size; 705 706 devdata->max_sync_sg = min(comp_sg_limit, decomp_sg_limit); 707 if (devdata->max_sync_sg < 1) { 708 dev_err(devdata->dev, "%s: hardware max sg size (%u) is " 709 "less than the driver minimum, unable to use " 710 "the hardware device\n", 711 __func__, devdata->max_sync_sg); 712 ret = -EINVAL; 713 goto out; 714 } 715 716out: 717 return ret; 718} 719 720/** 721 * nx842_OF_upd -- Handle OF properties updates for the device. 722 * 723 * Set all properties from the OF tree. Optionally, a new property 724 * can be provided by the @new_prop pointer to overwrite an existing value. 725 * The device will remain disabled until all values are valid, this function 726 * will return an error for updates unless all values are valid. 727 * 728 * @new_prop: If not NULL, this property is being updated. If NULL, update 729 * all properties from the current values in the OF tree. 730 * 731 * Returns: 732 * 0 - Success 733 * -ENOMEM - Could not allocate memory for new devdata structure 734 * -EINVAL - property value not found, new_prop is not a recognized 735 * property for the device or property value is not valid. 736 * -ENODEV - Device is not available 737 */ 738static int nx842_OF_upd(struct property *new_prop) 739{ 740 struct nx842_devdata *old_devdata = NULL; 741 struct nx842_devdata *new_devdata = NULL; 742 struct device_node *of_node = NULL; 743 struct property *status = NULL; 744 struct property *maxsglen = NULL; 745 struct property *maxsyncop = NULL; 746 int ret = 0; 747 unsigned long flags; 748 749 new_devdata = kzalloc(sizeof(*new_devdata), GFP_NOFS); 750 if (!new_devdata) 751 return -ENOMEM; 752 753 spin_lock_irqsave(&devdata_mutex, flags); 754 old_devdata = rcu_dereference_check(devdata, 755 lockdep_is_held(&devdata_mutex)); 756 if (old_devdata) 757 of_node = old_devdata->dev->of_node; 758 759 if (!old_devdata || !of_node) { 760 pr_err("%s: device is not available\n", __func__); 761 spin_unlock_irqrestore(&devdata_mutex, flags); 762 kfree(new_devdata); 763 return -ENODEV; 764 } 765 766 memcpy(new_devdata, old_devdata, sizeof(*old_devdata)); 767 new_devdata->counters = old_devdata->counters; 768 769 /* Set ptrs for existing properties */ 770 status = of_find_property(of_node, "status", NULL); 771 maxsglen = of_find_property(of_node, "ibm,max-sg-len", NULL); 772 maxsyncop = of_find_property(of_node, "ibm,max-sync-cop", NULL); 773 if (!status || !maxsglen || !maxsyncop) { 774 dev_err(old_devdata->dev, "%s: Could not locate device properties\n", __func__); 775 ret = -EINVAL; 776 goto error_out; 777 } 778 779 /* 780 * If this is a property update, there are only certain properties that 781 * we care about. Bail if it isn't in the below list 782 */ 783 if (new_prop && (strncmp(new_prop->name, "status", new_prop->length) || 784 strncmp(new_prop->name, "ibm,max-sg-len", new_prop->length) || 785 strncmp(new_prop->name, "ibm,max-sync-cop", new_prop->length))) 786 goto out; 787 788 /* Perform property updates */ 789 ret = nx842_OF_upd_status(new_devdata, status); 790 if (ret) 791 goto error_out; 792 793 ret = nx842_OF_upd_maxsglen(new_devdata, maxsglen); 794 if (ret) 795 goto error_out; 796 797 ret = nx842_OF_upd_maxsyncop(new_devdata, maxsyncop); 798 if (ret) 799 goto error_out; 800 801out: 802 dev_info(old_devdata->dev, "%s: max_sync_size new:%u old:%u\n", 803 __func__, new_devdata->max_sync_size, 804 old_devdata->max_sync_size); 805 dev_info(old_devdata->dev, "%s: max_sync_sg new:%u old:%u\n", 806 __func__, new_devdata->max_sync_sg, 807 old_devdata->max_sync_sg); 808 dev_info(old_devdata->dev, "%s: max_sg_len new:%u old:%u\n", 809 __func__, new_devdata->max_sg_len, 810 old_devdata->max_sg_len); 811 812 rcu_assign_pointer(devdata, new_devdata); 813 spin_unlock_irqrestore(&devdata_mutex, flags); 814 synchronize_rcu(); 815 dev_set_drvdata(new_devdata->dev, new_devdata); 816 kfree(old_devdata); 817 return 0; 818 819error_out: 820 if (new_devdata) { 821 dev_info(old_devdata->dev, "%s: device disabled\n", __func__); 822 nx842_OF_set_defaults(new_devdata); 823 rcu_assign_pointer(devdata, new_devdata); 824 spin_unlock_irqrestore(&devdata_mutex, flags); 825 synchronize_rcu(); 826 dev_set_drvdata(new_devdata->dev, new_devdata); 827 kfree(old_devdata); 828 } else { 829 dev_err(old_devdata->dev, "%s: could not update driver from hardware\n", __func__); 830 spin_unlock_irqrestore(&devdata_mutex, flags); 831 } 832 833 if (!ret) 834 ret = -EINVAL; 835 return ret; 836} 837 838/** 839 * nx842_OF_notifier - Process updates to OF properties for the device 840 * 841 * @np: notifier block 842 * @action: notifier action 843 * @data: struct of_reconfig_data pointer 844 * 845 * Returns: 846 * NOTIFY_OK on success 847 * NOTIFY_BAD encoded with error number on failure, use 848 * notifier_to_errno() to decode this value 849 */ 850static int nx842_OF_notifier(struct notifier_block *np, unsigned long action, 851 void *data) 852{ 853 struct of_reconfig_data *upd = data; 854 struct nx842_devdata *local_devdata; 855 struct device_node *node = NULL; 856 857 rcu_read_lock(); 858 local_devdata = rcu_dereference(devdata); 859 if (local_devdata) 860 node = local_devdata->dev->of_node; 861 862 if (local_devdata && 863 action == OF_RECONFIG_UPDATE_PROPERTY && 864 !strcmp(upd->dn->name, node->name)) { 865 rcu_read_unlock(); 866 nx842_OF_upd(upd->prop); 867 } else 868 rcu_read_unlock(); 869 870 return NOTIFY_OK; 871} 872 873static struct notifier_block nx842_of_nb = { 874 .notifier_call = nx842_OF_notifier, 875}; 876 877#define nx842_counter_read(_name) \ 878static ssize_t nx842_##_name##_show(struct device *dev, \ 879 struct device_attribute *attr, \ 880 char *buf) { \ 881 struct nx842_devdata *local_devdata; \ 882 int p = 0; \ 883 rcu_read_lock(); \ 884 local_devdata = rcu_dereference(devdata); \ 885 if (local_devdata) \ 886 p = snprintf(buf, PAGE_SIZE, "%lld\n", \ 887 atomic64_read(&local_devdata->counters->_name)); \ 888 rcu_read_unlock(); \ 889 return p; \ 890} 891 892#define NX842DEV_COUNTER_ATTR_RO(_name) \ 893 nx842_counter_read(_name); \ 894 static struct device_attribute dev_attr_##_name = __ATTR(_name, \ 895 0444, \ 896 nx842_##_name##_show,\ 897 NULL); 898 899NX842DEV_COUNTER_ATTR_RO(comp_complete); 900NX842DEV_COUNTER_ATTR_RO(comp_failed); 901NX842DEV_COUNTER_ATTR_RO(decomp_complete); 902NX842DEV_COUNTER_ATTR_RO(decomp_failed); 903NX842DEV_COUNTER_ATTR_RO(swdecomp); 904 905static ssize_t nx842_timehist_show(struct device *, 906 struct device_attribute *, char *); 907 908static struct device_attribute dev_attr_comp_times = __ATTR(comp_times, 0444, 909 nx842_timehist_show, NULL); 910static struct device_attribute dev_attr_decomp_times = __ATTR(decomp_times, 911 0444, nx842_timehist_show, NULL); 912 913static ssize_t nx842_timehist_show(struct device *dev, 914 struct device_attribute *attr, char *buf) { 915 char *p = buf; 916 struct nx842_devdata *local_devdata; 917 atomic64_t *times; 918 int bytes_remain = PAGE_SIZE; 919 int bytes; 920 int i; 921 922 rcu_read_lock(); 923 local_devdata = rcu_dereference(devdata); 924 if (!local_devdata) { 925 rcu_read_unlock(); 926 return 0; 927 } 928 929 if (attr == &dev_attr_comp_times) 930 times = local_devdata->counters->comp_times; 931 else if (attr == &dev_attr_decomp_times) 932 times = local_devdata->counters->decomp_times; 933 else { 934 rcu_read_unlock(); 935 return 0; 936 } 937 938 for (i = 0; i < (NX842_HIST_SLOTS - 2); i++) { 939 bytes = snprintf(p, bytes_remain, "%u-%uus:\t%lld\n", 940 i ? (2<<(i-1)) : 0, (2<<i)-1, 941 atomic64_read(×[i])); 942 bytes_remain -= bytes; 943 p += bytes; 944 } 945 /* The last bucket holds everything over 946 * 2<<(NX842_HIST_SLOTS - 2) us */ 947 bytes = snprintf(p, bytes_remain, "%uus - :\t%lld\n", 948 2<<(NX842_HIST_SLOTS - 2), 949 atomic64_read(×[(NX842_HIST_SLOTS - 1)])); 950 p += bytes; 951 952 rcu_read_unlock(); 953 return p - buf; 954} 955 956static struct attribute *nx842_sysfs_entries[] = { 957 &dev_attr_comp_complete.attr, 958 &dev_attr_comp_failed.attr, 959 &dev_attr_decomp_complete.attr, 960 &dev_attr_decomp_failed.attr, 961 &dev_attr_swdecomp.attr, 962 &dev_attr_comp_times.attr, 963 &dev_attr_decomp_times.attr, 964 NULL, 965}; 966 967static const struct attribute_group nx842_attribute_group = { 968 .name = NULL, /* put in device directory */ 969 .attrs = nx842_sysfs_entries, 970}; 971 972#define nxcop_caps_read(_name) \ 973static ssize_t nxcop_##_name##_show(struct device *dev, \ 974 struct device_attribute *attr, char *buf) \ 975{ \ 976 return sprintf(buf, "%lld\n", nx_cop_caps._name); \ 977} 978 979#define NXCT_ATTR_RO(_name) \ 980 nxcop_caps_read(_name); \ 981 static struct device_attribute dev_attr_##_name = __ATTR(_name, \ 982 0444, \ 983 nxcop_##_name##_show, \ 984 NULL); 985 986NXCT_ATTR_RO(req_max_processed_len); 987NXCT_ATTR_RO(min_compress_len); 988NXCT_ATTR_RO(min_decompress_len); 989 990static struct attribute *nxcop_caps_sysfs_entries[] = { 991 &dev_attr_req_max_processed_len.attr, 992 &dev_attr_min_compress_len.attr, 993 &dev_attr_min_decompress_len.attr, 994 NULL, 995}; 996 997static const struct attribute_group nxcop_caps_attr_group = { 998 .name = "nx_gzip_caps", 999 .attrs = nxcop_caps_sysfs_entries, 1000}; 1001 1002static struct nx842_driver nx842_pseries_driver = { 1003 .name = KBUILD_MODNAME, 1004 .owner = THIS_MODULE, 1005 .workmem_size = sizeof(struct nx842_workmem), 1006 .constraints = &nx842_pseries_constraints, 1007 .compress = nx842_pseries_compress, 1008 .decompress = nx842_pseries_decompress, 1009}; 1010 1011static int nx842_pseries_crypto_init(struct crypto_tfm *tfm) 1012{ 1013 return nx842_crypto_init(tfm, &nx842_pseries_driver); 1014} 1015 1016static struct crypto_alg nx842_pseries_alg = { 1017 .cra_name = "842", 1018 .cra_driver_name = "842-nx", 1019 .cra_priority = 300, 1020 .cra_flags = CRYPTO_ALG_TYPE_COMPRESS, 1021 .cra_ctxsize = sizeof(struct nx842_crypto_ctx), 1022 .cra_module = THIS_MODULE, 1023 .cra_init = nx842_pseries_crypto_init, 1024 .cra_exit = nx842_crypto_exit, 1025 .cra_u = { .compress = { 1026 .coa_compress = nx842_crypto_compress, 1027 .coa_decompress = nx842_crypto_decompress } } 1028}; 1029 1030static int nx842_probe(struct vio_dev *viodev, 1031 const struct vio_device_id *id) 1032{ 1033 struct nx842_devdata *old_devdata, *new_devdata = NULL; 1034 unsigned long flags; 1035 int ret = 0; 1036 1037 new_devdata = kzalloc(sizeof(*new_devdata), GFP_NOFS); 1038 if (!new_devdata) 1039 return -ENOMEM; 1040 1041 new_devdata->counters = kzalloc(sizeof(*new_devdata->counters), 1042 GFP_NOFS); 1043 if (!new_devdata->counters) { 1044 kfree(new_devdata); 1045 return -ENOMEM; 1046 } 1047 1048 spin_lock_irqsave(&devdata_mutex, flags); 1049 old_devdata = rcu_dereference_check(devdata, 1050 lockdep_is_held(&devdata_mutex)); 1051 1052 if (old_devdata && old_devdata->vdev != NULL) { 1053 dev_err(&viodev->dev, "%s: Attempt to register more than one instance of the hardware\n", __func__); 1054 ret = -1; 1055 goto error_unlock; 1056 } 1057 1058 dev_set_drvdata(&viodev->dev, NULL); 1059 1060 new_devdata->vdev = viodev; 1061 new_devdata->dev = &viodev->dev; 1062 nx842_OF_set_defaults(new_devdata); 1063 1064 rcu_assign_pointer(devdata, new_devdata); 1065 spin_unlock_irqrestore(&devdata_mutex, flags); 1066 synchronize_rcu(); 1067 kfree(old_devdata); 1068 1069 of_reconfig_notifier_register(&nx842_of_nb); 1070 1071 ret = nx842_OF_upd(NULL); 1072 if (ret) 1073 goto error; 1074 1075 ret = crypto_register_alg(&nx842_pseries_alg); 1076 if (ret) { 1077 dev_err(&viodev->dev, "could not register comp alg: %d\n", ret); 1078 goto error; 1079 } 1080 1081 rcu_read_lock(); 1082 dev_set_drvdata(&viodev->dev, rcu_dereference(devdata)); 1083 rcu_read_unlock(); 1084 1085 if (sysfs_create_group(&viodev->dev.kobj, &nx842_attribute_group)) { 1086 dev_err(&viodev->dev, "could not create sysfs device attributes\n"); 1087 ret = -1; 1088 goto error; 1089 } 1090 1091 if (caps_feat) { 1092 if (sysfs_create_group(&viodev->dev.kobj, 1093 &nxcop_caps_attr_group)) { 1094 dev_err(&viodev->dev, 1095 "Could not create sysfs NX capability entries\n"); 1096 ret = -1; 1097 goto error; 1098 } 1099 } 1100 1101 return 0; 1102 1103error_unlock: 1104 spin_unlock_irqrestore(&devdata_mutex, flags); 1105 if (new_devdata) 1106 kfree(new_devdata->counters); 1107 kfree(new_devdata); 1108error: 1109 return ret; 1110} 1111 1112static void nx842_remove(struct vio_dev *viodev) 1113{ 1114 struct nx842_devdata *old_devdata; 1115 unsigned long flags; 1116 1117 pr_info("Removing IBM Power 842 compression device\n"); 1118 sysfs_remove_group(&viodev->dev.kobj, &nx842_attribute_group); 1119 1120 if (caps_feat) 1121 sysfs_remove_group(&viodev->dev.kobj, &nxcop_caps_attr_group); 1122 1123 crypto_unregister_alg(&nx842_pseries_alg); 1124 1125 spin_lock_irqsave(&devdata_mutex, flags); 1126 old_devdata = rcu_dereference_check(devdata, 1127 lockdep_is_held(&devdata_mutex)); 1128 of_reconfig_notifier_unregister(&nx842_of_nb); 1129 RCU_INIT_POINTER(devdata, NULL); 1130 spin_unlock_irqrestore(&devdata_mutex, flags); 1131 synchronize_rcu(); 1132 dev_set_drvdata(&viodev->dev, NULL); 1133 if (old_devdata) 1134 kfree(old_devdata->counters); 1135 kfree(old_devdata); 1136} 1137 1138/* 1139 * Get NX capabilities from the hypervisor. 1140 * Only NXGZIP capabilities are provided by the hypersvisor right 1141 * now and these values are available to user space with sysfs. 1142 */ 1143static void __init nxcop_get_capabilities(void) 1144{ 1145 struct hv_vas_all_caps *hv_caps; 1146 struct hv_nx_cop_caps *hv_nxc; 1147 int rc; 1148 1149 hv_caps = kmalloc(sizeof(*hv_caps), GFP_KERNEL); 1150 if (!hv_caps) 1151 return; 1152 /* 1153 * Get NX overall capabilities with feature type=0 1154 */ 1155 rc = h_query_vas_capabilities(H_QUERY_NX_CAPABILITIES, 0, 1156 (u64)virt_to_phys(hv_caps)); 1157 if (rc) 1158 goto out; 1159 1160 caps_feat = be64_to_cpu(hv_caps->feat_type); 1161 /* 1162 * NX-GZIP feature available 1163 */ 1164 if (caps_feat & VAS_NX_GZIP_FEAT_BIT) { 1165 hv_nxc = kmalloc(sizeof(*hv_nxc), GFP_KERNEL); 1166 if (!hv_nxc) 1167 goto out; 1168 /* 1169 * Get capabilities for NX-GZIP feature 1170 */ 1171 rc = h_query_vas_capabilities(H_QUERY_NX_CAPABILITIES, 1172 VAS_NX_GZIP_FEAT, 1173 (u64)virt_to_phys(hv_nxc)); 1174 } else { 1175 pr_err("NX-GZIP feature is not available\n"); 1176 rc = -EINVAL; 1177 } 1178 1179 if (!rc) { 1180 nx_cop_caps.descriptor = be64_to_cpu(hv_nxc->descriptor); 1181 nx_cop_caps.req_max_processed_len = 1182 be64_to_cpu(hv_nxc->req_max_processed_len); 1183 nx_cop_caps.min_compress_len = 1184 be64_to_cpu(hv_nxc->min_compress_len); 1185 nx_cop_caps.min_decompress_len = 1186 be64_to_cpu(hv_nxc->min_decompress_len); 1187 } else { 1188 caps_feat = 0; 1189 } 1190 1191 kfree(hv_nxc); 1192out: 1193 kfree(hv_caps); 1194} 1195 1196static const struct vio_device_id nx842_vio_driver_ids[] = { 1197 {"ibm,compression-v1", "ibm,compression"}, 1198 {"", ""}, 1199}; 1200MODULE_DEVICE_TABLE(vio, nx842_vio_driver_ids); 1201 1202static struct vio_driver nx842_vio_driver = { 1203 .name = KBUILD_MODNAME, 1204 .probe = nx842_probe, 1205 .remove = nx842_remove, 1206 .get_desired_dma = nx842_get_desired_dma, 1207 .id_table = nx842_vio_driver_ids, 1208}; 1209 1210static int __init nx842_pseries_init(void) 1211{ 1212 struct nx842_devdata *new_devdata; 1213 struct device_node *np; 1214 int ret; 1215 1216 np = of_find_compatible_node(NULL, NULL, "ibm,compression"); 1217 if (!np) 1218 return -ENODEV; 1219 of_node_put(np); 1220 1221 RCU_INIT_POINTER(devdata, NULL); 1222 new_devdata = kzalloc(sizeof(*new_devdata), GFP_KERNEL); 1223 if (!new_devdata) 1224 return -ENOMEM; 1225 1226 RCU_INIT_POINTER(devdata, new_devdata); 1227 /* 1228 * Get NX capabilities from the hypervisor. 1229 */ 1230 nxcop_get_capabilities(); 1231 1232 ret = vio_register_driver(&nx842_vio_driver); 1233 if (ret) { 1234 pr_err("Could not register VIO driver %d\n", ret); 1235 1236 kfree(new_devdata); 1237 return ret; 1238 } 1239 1240 ret = vas_register_api_pseries(THIS_MODULE, VAS_COP_TYPE_GZIP, 1241 "nx-gzip"); 1242 1243 if (ret) 1244 pr_err("NX-GZIP is not supported. Returned=%d\n", ret); 1245 1246 return 0; 1247} 1248 1249module_init(nx842_pseries_init); 1250 1251static void __exit nx842_pseries_exit(void) 1252{ 1253 struct nx842_devdata *old_devdata; 1254 unsigned long flags; 1255 1256 vas_unregister_api_pseries(); 1257 1258 crypto_unregister_alg(&nx842_pseries_alg); 1259 1260 spin_lock_irqsave(&devdata_mutex, flags); 1261 old_devdata = rcu_dereference_check(devdata, 1262 lockdep_is_held(&devdata_mutex)); 1263 RCU_INIT_POINTER(devdata, NULL); 1264 spin_unlock_irqrestore(&devdata_mutex, flags); 1265 synchronize_rcu(); 1266 if (old_devdata && old_devdata->dev) 1267 dev_set_drvdata(old_devdata->dev, NULL); 1268 kfree(old_devdata); 1269 vio_unregister_driver(&nx842_vio_driver); 1270} 1271 1272module_exit(nx842_pseries_exit); 1273 1274