1// SPDX-License-Identifier: GPL-2.0-or-later 2/* 3 * Driver for IBM Power 842 compression accelerator 4 * 5 * Copyright (C) IBM Corporation, 2012 6 * 7 * Authors: Robert Jennings <rcj@linux.vnet.ibm.com> 8 * Seth Jennings <sjenning@linux.vnet.ibm.com> 9 */ 10 11#include <asm/vio.h> 12 13#include "nx-842.h" 14#include "nx_csbcpb.h" /* struct nx_csbcpb */ 15 16MODULE_LICENSE("GPL"); 17MODULE_AUTHOR("Robert Jennings <rcj@linux.vnet.ibm.com>"); 18MODULE_DESCRIPTION("842 H/W Compression driver for IBM Power processors"); 19MODULE_ALIAS_CRYPTO("842"); 20MODULE_ALIAS_CRYPTO("842-nx"); 21 22static struct nx842_constraints nx842_pseries_constraints = { 23 .alignment = DDE_BUFFER_ALIGN, 24 .multiple = DDE_BUFFER_LAST_MULT, 25 .minimum = DDE_BUFFER_LAST_MULT, 26 .maximum = PAGE_SIZE, /* dynamic, max_sync_size */ 27}; 28 29static int check_constraints(unsigned long buf, unsigned int *len, bool in) 30{ 31 if (!IS_ALIGNED(buf, nx842_pseries_constraints.alignment)) { 32 pr_debug("%s buffer 0x%lx not aligned to 0x%x\n", 33 in ? "input" : "output", buf, 34 nx842_pseries_constraints.alignment); 35 return -EINVAL; 36 } 37 if (*len % nx842_pseries_constraints.multiple) { 38 pr_debug("%s buffer len 0x%x not multiple of 0x%x\n", 39 in ? "input" : "output", *len, 40 nx842_pseries_constraints.multiple); 41 if (in) 42 return -EINVAL; 43 *len = round_down(*len, nx842_pseries_constraints.multiple); 44 } 45 if (*len < nx842_pseries_constraints.minimum) { 46 pr_debug("%s buffer len 0x%x under minimum 0x%x\n", 47 in ? "input" : "output", *len, 48 nx842_pseries_constraints.minimum); 49 return -EINVAL; 50 } 51 if (*len > nx842_pseries_constraints.maximum) { 52 pr_debug("%s buffer len 0x%x over maximum 0x%x\n", 53 in ? "input" : "output", *len, 54 nx842_pseries_constraints.maximum); 55 if (in) 56 return -EINVAL; 57 *len = nx842_pseries_constraints.maximum; 58 } 59 return 0; 60} 61 62/* I assume we need to align the CSB? */ 63#define WORKMEM_ALIGN (256) 64 65struct nx842_workmem { 66 /* scatterlist */ 67 char slin[4096]; 68 char slout[4096]; 69 /* coprocessor status/parameter block */ 70 struct nx_csbcpb csbcpb; 71 72 char padding[WORKMEM_ALIGN]; 73} __aligned(WORKMEM_ALIGN); 74 75/* Macros for fields within nx_csbcpb */ 76/* Check the valid bit within the csbcpb valid field */ 77#define NX842_CSBCBP_VALID_CHK(x) (x & BIT_MASK(7)) 78 79/* CE macros operate on the completion_extension field bits in the csbcpb. 80 * CE0 0=full completion, 1=partial completion 81 * CE1 0=CE0 indicates completion, 1=termination (output may be modified) 82 * CE2 0=processed_bytes is source bytes, 1=processed_bytes is target bytes */ 83#define NX842_CSBCPB_CE0(x) (x & BIT_MASK(7)) 84#define NX842_CSBCPB_CE1(x) (x & BIT_MASK(6)) 85#define NX842_CSBCPB_CE2(x) (x & BIT_MASK(5)) 86 87/* The NX unit accepts data only on 4K page boundaries */ 88#define NX842_HW_PAGE_SIZE (4096) 89#define NX842_HW_PAGE_MASK (~(NX842_HW_PAGE_SIZE-1)) 90 91struct ibm_nx842_counters { 92 atomic64_t comp_complete; 93 atomic64_t comp_failed; 94 atomic64_t decomp_complete; 95 atomic64_t decomp_failed; 96 atomic64_t swdecomp; 97 atomic64_t comp_times[32]; 98 atomic64_t decomp_times[32]; 99}; 100 101static struct nx842_devdata { 102 struct vio_dev *vdev; 103 struct device *dev; 104 struct ibm_nx842_counters *counters; 105 unsigned int max_sg_len; 106 unsigned int max_sync_size; 107 unsigned int max_sync_sg; 108} __rcu *devdata; 109static DEFINE_SPINLOCK(devdata_mutex); 110 111#define NX842_COUNTER_INC(_x) \ 112static inline void nx842_inc_##_x( \ 113 const struct nx842_devdata *dev) { \ 114 if (dev) \ 115 atomic64_inc(&dev->counters->_x); \ 116} 117NX842_COUNTER_INC(comp_complete); 118NX842_COUNTER_INC(comp_failed); 119NX842_COUNTER_INC(decomp_complete); 120NX842_COUNTER_INC(decomp_failed); 121NX842_COUNTER_INC(swdecomp); 122 123#define NX842_HIST_SLOTS 16 124 125static void ibm_nx842_incr_hist(atomic64_t *times, unsigned int time) 126{ 127 int bucket = fls(time); 128 129 if (bucket) 130 bucket = min((NX842_HIST_SLOTS - 1), bucket - 1); 131 132 atomic64_inc(×[bucket]); 133} 134 135/* NX unit operation flags */ 136#define NX842_OP_COMPRESS 0x0 137#define NX842_OP_CRC 0x1 138#define NX842_OP_DECOMPRESS 0x2 139#define NX842_OP_COMPRESS_CRC (NX842_OP_COMPRESS | NX842_OP_CRC) 140#define NX842_OP_DECOMPRESS_CRC (NX842_OP_DECOMPRESS | NX842_OP_CRC) 141#define NX842_OP_ASYNC (1<<23) 142#define NX842_OP_NOTIFY (1<<22) 143#define NX842_OP_NOTIFY_INT(x) ((x & 0xff)<<8) 144 145static unsigned long nx842_get_desired_dma(struct vio_dev *viodev) 146{ 147 /* No use of DMA mappings within the driver. */ 148 return 0; 149} 150 151struct nx842_slentry { 152 __be64 ptr; /* Real address (use __pa()) */ 153 __be64 len; 154}; 155 156/* pHyp scatterlist entry */ 157struct nx842_scatterlist { 158 int entry_nr; /* number of slentries */ 159 struct nx842_slentry *entries; /* ptr to array of slentries */ 160}; 161 162/* Does not include sizeof(entry_nr) in the size */ 163static inline unsigned long nx842_get_scatterlist_size( 164 struct nx842_scatterlist *sl) 165{ 166 return sl->entry_nr * sizeof(struct nx842_slentry); 167} 168 169static int nx842_build_scatterlist(unsigned long buf, int len, 170 struct nx842_scatterlist *sl) 171{ 172 unsigned long entrylen; 173 struct nx842_slentry *entry; 174 175 sl->entry_nr = 0; 176 177 entry = sl->entries; 178 while (len) { 179 entry->ptr = cpu_to_be64(nx842_get_pa((void *)buf)); 180 entrylen = min_t(int, len, 181 LEN_ON_SIZE(buf, NX842_HW_PAGE_SIZE)); 182 entry->len = cpu_to_be64(entrylen); 183 184 len -= entrylen; 185 buf += entrylen; 186 187 sl->entry_nr++; 188 entry++; 189 } 190 191 return 0; 192} 193 194static int nx842_validate_result(struct device *dev, 195 struct cop_status_block *csb) 196{ 197 /* The csb must be valid after returning from vio_h_cop_sync */ 198 if (!NX842_CSBCBP_VALID_CHK(csb->valid)) { 199 dev_err(dev, "%s: cspcbp not valid upon completion.\n", 200 __func__); 201 dev_dbg(dev, "valid:0x%02x cs:0x%02x cc:0x%02x ce:0x%02x\n", 202 csb->valid, 203 csb->crb_seq_number, 204 csb->completion_code, 205 csb->completion_extension); 206 dev_dbg(dev, "processed_bytes:%d address:0x%016lx\n", 207 be32_to_cpu(csb->processed_byte_count), 208 (unsigned long)be64_to_cpu(csb->address)); 209 return -EIO; 210 } 211 212 /* Check return values from the hardware in the CSB */ 213 switch (csb->completion_code) { 214 case 0: /* Completed without error */ 215 break; 216 case 64: /* Compression ok, but output larger than input */ 217 dev_dbg(dev, "%s: output size larger than input size\n", 218 __func__); 219 break; 220 case 13: /* Output buffer too small */ 221 dev_dbg(dev, "%s: Out of space in output buffer\n", 222 __func__); 223 return -ENOSPC; 224 case 65: /* Calculated CRC doesn't match the passed value */ 225 dev_dbg(dev, "%s: CRC mismatch for decompression\n", 226 __func__); 227 return -EINVAL; 228 case 66: /* Input data contains an illegal template field */ 229 case 67: /* Template indicates data past the end of the input stream */ 230 dev_dbg(dev, "%s: Bad data for decompression (code:%d)\n", 231 __func__, csb->completion_code); 232 return -EINVAL; 233 default: 234 dev_dbg(dev, "%s: Unspecified error (code:%d)\n", 235 __func__, csb->completion_code); 236 return -EIO; 237 } 238 239 /* Hardware sanity check */ 240 if (!NX842_CSBCPB_CE2(csb->completion_extension)) { 241 dev_err(dev, "%s: No error returned by hardware, but " 242 "data returned is unusable, contact support.\n" 243 "(Additional info: csbcbp->processed bytes " 244 "does not specify processed bytes for the " 245 "target buffer.)\n", __func__); 246 return -EIO; 247 } 248 249 return 0; 250} 251 252/** 253 * nx842_pseries_compress - Compress data using the 842 algorithm 254 * 255 * Compression provide by the NX842 coprocessor on IBM Power systems. 256 * The input buffer is compressed and the result is stored in the 257 * provided output buffer. 258 * 259 * Upon return from this function @outlen contains the length of the 260 * compressed data. If there is an error then @outlen will be 0 and an 261 * error will be specified by the return code from this function. 262 * 263 * @in: Pointer to input buffer 264 * @inlen: Length of input buffer 265 * @out: Pointer to output buffer 266 * @outlen: Length of output buffer 267 * @wrkmem: ptr to buffer for working memory, size determined by 268 * nx842_pseries_driver.workmem_size 269 * 270 * Returns: 271 * 0 Success, output of length @outlen stored in the buffer at @out 272 * -ENOMEM Unable to allocate internal buffers 273 * -ENOSPC Output buffer is to small 274 * -EIO Internal error 275 * -ENODEV Hardware unavailable 276 */ 277static int nx842_pseries_compress(const unsigned char *in, unsigned int inlen, 278 unsigned char *out, unsigned int *outlen, 279 void *wmem) 280{ 281 struct nx842_devdata *local_devdata; 282 struct device *dev = NULL; 283 struct nx842_workmem *workmem; 284 struct nx842_scatterlist slin, slout; 285 struct nx_csbcpb *csbcpb; 286 int ret = 0; 287 unsigned long inbuf, outbuf; 288 struct vio_pfo_op op = { 289 .done = NULL, 290 .handle = 0, 291 .timeout = 0, 292 }; 293 unsigned long start = get_tb(); 294 295 inbuf = (unsigned long)in; 296 if (check_constraints(inbuf, &inlen, true)) 297 return -EINVAL; 298 299 outbuf = (unsigned long)out; 300 if (check_constraints(outbuf, outlen, false)) 301 return -EINVAL; 302 303 rcu_read_lock(); 304 local_devdata = rcu_dereference(devdata); 305 if (!local_devdata || !local_devdata->dev) { 306 rcu_read_unlock(); 307 return -ENODEV; 308 } 309 dev = local_devdata->dev; 310 311 /* Init scatterlist */ 312 workmem = PTR_ALIGN(wmem, WORKMEM_ALIGN); 313 slin.entries = (struct nx842_slentry *)workmem->slin; 314 slout.entries = (struct nx842_slentry *)workmem->slout; 315 316 /* Init operation */ 317 op.flags = NX842_OP_COMPRESS_CRC; 318 csbcpb = &workmem->csbcpb; 319 memset(csbcpb, 0, sizeof(*csbcpb)); 320 op.csbcpb = nx842_get_pa(csbcpb); 321 322 if ((inbuf & NX842_HW_PAGE_MASK) == 323 ((inbuf + inlen - 1) & NX842_HW_PAGE_MASK)) { 324 /* Create direct DDE */ 325 op.in = nx842_get_pa((void *)inbuf); 326 op.inlen = inlen; 327 } else { 328 /* Create indirect DDE (scatterlist) */ 329 nx842_build_scatterlist(inbuf, inlen, &slin); 330 op.in = nx842_get_pa(slin.entries); 331 op.inlen = -nx842_get_scatterlist_size(&slin); 332 } 333 334 if ((outbuf & NX842_HW_PAGE_MASK) == 335 ((outbuf + *outlen - 1) & NX842_HW_PAGE_MASK)) { 336 /* Create direct DDE */ 337 op.out = nx842_get_pa((void *)outbuf); 338 op.outlen = *outlen; 339 } else { 340 /* Create indirect DDE (scatterlist) */ 341 nx842_build_scatterlist(outbuf, *outlen, &slout); 342 op.out = nx842_get_pa(slout.entries); 343 op.outlen = -nx842_get_scatterlist_size(&slout); 344 } 345 346 dev_dbg(dev, "%s: op.in %lx op.inlen %ld op.out %lx op.outlen %ld\n", 347 __func__, (unsigned long)op.in, (long)op.inlen, 348 (unsigned long)op.out, (long)op.outlen); 349 350 /* Send request to pHyp */ 351 ret = vio_h_cop_sync(local_devdata->vdev, &op); 352 353 /* Check for pHyp error */ 354 if (ret) { 355 dev_dbg(dev, "%s: vio_h_cop_sync error (ret=%d, hret=%ld)\n", 356 __func__, ret, op.hcall_err); 357 ret = -EIO; 358 goto unlock; 359 } 360 361 /* Check for hardware error */ 362 ret = nx842_validate_result(dev, &csbcpb->csb); 363 if (ret) 364 goto unlock; 365 366 *outlen = be32_to_cpu(csbcpb->csb.processed_byte_count); 367 dev_dbg(dev, "%s: processed_bytes=%d\n", __func__, *outlen); 368 369unlock: 370 if (ret) 371 nx842_inc_comp_failed(local_devdata); 372 else { 373 nx842_inc_comp_complete(local_devdata); 374 ibm_nx842_incr_hist(local_devdata->counters->comp_times, 375 (get_tb() - start) / tb_ticks_per_usec); 376 } 377 rcu_read_unlock(); 378 return ret; 379} 380 381/** 382 * nx842_pseries_decompress - Decompress data using the 842 algorithm 383 * 384 * Decompression provide by the NX842 coprocessor on IBM Power systems. 385 * The input buffer is decompressed and the result is stored in the 386 * provided output buffer. The size allocated to the output buffer is 387 * provided by the caller of this function in @outlen. Upon return from 388 * this function @outlen contains the length of the decompressed data. 389 * If there is an error then @outlen will be 0 and an error will be 390 * specified by the return code from this function. 391 * 392 * @in: Pointer to input buffer 393 * @inlen: Length of input buffer 394 * @out: Pointer to output buffer 395 * @outlen: Length of output buffer 396 * @wrkmem: ptr to buffer for working memory, size determined by 397 * nx842_pseries_driver.workmem_size 398 * 399 * Returns: 400 * 0 Success, output of length @outlen stored in the buffer at @out 401 * -ENODEV Hardware decompression device is unavailable 402 * -ENOMEM Unable to allocate internal buffers 403 * -ENOSPC Output buffer is to small 404 * -EINVAL Bad input data encountered when attempting decompress 405 * -EIO Internal error 406 */ 407static int nx842_pseries_decompress(const unsigned char *in, unsigned int inlen, 408 unsigned char *out, unsigned int *outlen, 409 void *wmem) 410{ 411 struct nx842_devdata *local_devdata; 412 struct device *dev = NULL; 413 struct nx842_workmem *workmem; 414 struct nx842_scatterlist slin, slout; 415 struct nx_csbcpb *csbcpb; 416 int ret = 0; 417 unsigned long inbuf, outbuf; 418 struct vio_pfo_op op = { 419 .done = NULL, 420 .handle = 0, 421 .timeout = 0, 422 }; 423 unsigned long start = get_tb(); 424 425 /* Ensure page alignment and size */ 426 inbuf = (unsigned long)in; 427 if (check_constraints(inbuf, &inlen, true)) 428 return -EINVAL; 429 430 outbuf = (unsigned long)out; 431 if (check_constraints(outbuf, outlen, false)) 432 return -EINVAL; 433 434 rcu_read_lock(); 435 local_devdata = rcu_dereference(devdata); 436 if (!local_devdata || !local_devdata->dev) { 437 rcu_read_unlock(); 438 return -ENODEV; 439 } 440 dev = local_devdata->dev; 441 442 workmem = PTR_ALIGN(wmem, WORKMEM_ALIGN); 443 444 /* Init scatterlist */ 445 slin.entries = (struct nx842_slentry *)workmem->slin; 446 slout.entries = (struct nx842_slentry *)workmem->slout; 447 448 /* Init operation */ 449 op.flags = NX842_OP_DECOMPRESS_CRC; 450 csbcpb = &workmem->csbcpb; 451 memset(csbcpb, 0, sizeof(*csbcpb)); 452 op.csbcpb = nx842_get_pa(csbcpb); 453 454 if ((inbuf & NX842_HW_PAGE_MASK) == 455 ((inbuf + inlen - 1) & NX842_HW_PAGE_MASK)) { 456 /* Create direct DDE */ 457 op.in = nx842_get_pa((void *)inbuf); 458 op.inlen = inlen; 459 } else { 460 /* Create indirect DDE (scatterlist) */ 461 nx842_build_scatterlist(inbuf, inlen, &slin); 462 op.in = nx842_get_pa(slin.entries); 463 op.inlen = -nx842_get_scatterlist_size(&slin); 464 } 465 466 if ((outbuf & NX842_HW_PAGE_MASK) == 467 ((outbuf + *outlen - 1) & NX842_HW_PAGE_MASK)) { 468 /* Create direct DDE */ 469 op.out = nx842_get_pa((void *)outbuf); 470 op.outlen = *outlen; 471 } else { 472 /* Create indirect DDE (scatterlist) */ 473 nx842_build_scatterlist(outbuf, *outlen, &slout); 474 op.out = nx842_get_pa(slout.entries); 475 op.outlen = -nx842_get_scatterlist_size(&slout); 476 } 477 478 dev_dbg(dev, "%s: op.in %lx op.inlen %ld op.out %lx op.outlen %ld\n", 479 __func__, (unsigned long)op.in, (long)op.inlen, 480 (unsigned long)op.out, (long)op.outlen); 481 482 /* Send request to pHyp */ 483 ret = vio_h_cop_sync(local_devdata->vdev, &op); 484 485 /* Check for pHyp error */ 486 if (ret) { 487 dev_dbg(dev, "%s: vio_h_cop_sync error (ret=%d, hret=%ld)\n", 488 __func__, ret, op.hcall_err); 489 goto unlock; 490 } 491 492 /* Check for hardware error */ 493 ret = nx842_validate_result(dev, &csbcpb->csb); 494 if (ret) 495 goto unlock; 496 497 *outlen = be32_to_cpu(csbcpb->csb.processed_byte_count); 498 499unlock: 500 if (ret) 501 /* decompress fail */ 502 nx842_inc_decomp_failed(local_devdata); 503 else { 504 nx842_inc_decomp_complete(local_devdata); 505 ibm_nx842_incr_hist(local_devdata->counters->decomp_times, 506 (get_tb() - start) / tb_ticks_per_usec); 507 } 508 509 rcu_read_unlock(); 510 return ret; 511} 512 513/** 514 * nx842_OF_set_defaults -- Set default (disabled) values for devdata 515 * 516 * @devdata - struct nx842_devdata to update 517 * 518 * Returns: 519 * 0 on success 520 * -ENOENT if @devdata ptr is NULL 521 */ 522static int nx842_OF_set_defaults(struct nx842_devdata *devdata) 523{ 524 if (devdata) { 525 devdata->max_sync_size = 0; 526 devdata->max_sync_sg = 0; 527 devdata->max_sg_len = 0; 528 return 0; 529 } else 530 return -ENOENT; 531} 532 533/** 534 * nx842_OF_upd_status -- Check the device info from OF status prop 535 * 536 * The status property indicates if the accelerator is enabled. If the 537 * device is in the OF tree it indicates that the hardware is present. 538 * The status field indicates if the device is enabled when the status 539 * is 'okay'. Otherwise the device driver will be disabled. 540 * 541 * @devdata: struct nx842_devdata to use for dev_info 542 * @prop: struct property point containing the maxsyncop for the update 543 * 544 * Returns: 545 * 0 - Device is available 546 * -ENODEV - Device is not available 547 */ 548static int nx842_OF_upd_status(struct nx842_devdata *devdata, 549 struct property *prop) 550{ 551 const char *status = (const char *)prop->value; 552 553 if (!strncmp(status, "okay", (size_t)prop->length)) 554 return 0; 555 if (!strncmp(status, "disabled", (size_t)prop->length)) 556 return -ENODEV; 557 dev_info(devdata->dev, "%s: unknown status '%s'\n", __func__, status); 558 559 return -EINVAL; 560} 561 562/** 563 * nx842_OF_upd_maxsglen -- Update the device info from OF maxsglen prop 564 * 565 * Definition of the 'ibm,max-sg-len' OF property: 566 * This field indicates the maximum byte length of a scatter list 567 * for the platform facility. It is a single cell encoded as with encode-int. 568 * 569 * Example: 570 * # od -x ibm,max-sg-len 571 * 0000000 0000 0ff0 572 * 573 * In this example, the maximum byte length of a scatter list is 574 * 0x0ff0 (4,080). 575 * 576 * @devdata - struct nx842_devdata to update 577 * @prop - struct property point containing the maxsyncop for the update 578 * 579 * Returns: 580 * 0 on success 581 * -EINVAL on failure 582 */ 583static int nx842_OF_upd_maxsglen(struct nx842_devdata *devdata, 584 struct property *prop) { 585 int ret = 0; 586 const unsigned int maxsglen = of_read_number(prop->value, 1); 587 588 if (prop->length != sizeof(maxsglen)) { 589 dev_err(devdata->dev, "%s: unexpected format for ibm,max-sg-len property\n", __func__); 590 dev_dbg(devdata->dev, "%s: ibm,max-sg-len is %d bytes long, expected %lu bytes\n", __func__, 591 prop->length, sizeof(maxsglen)); 592 ret = -EINVAL; 593 } else { 594 devdata->max_sg_len = min_t(unsigned int, 595 maxsglen, NX842_HW_PAGE_SIZE); 596 } 597 598 return ret; 599} 600 601/** 602 * nx842_OF_upd_maxsyncop -- Update the device info from OF maxsyncop prop 603 * 604 * Definition of the 'ibm,max-sync-cop' OF property: 605 * Two series of cells. The first series of cells represents the maximums 606 * that can be synchronously compressed. The second series of cells 607 * represents the maximums that can be synchronously decompressed. 608 * 1. The first cell in each series contains the count of the number of 609 * data length, scatter list elements pairs that follow – each being 610 * of the form 611 * a. One cell data byte length 612 * b. One cell total number of scatter list elements 613 * 614 * Example: 615 * # od -x ibm,max-sync-cop 616 * 0000000 0000 0001 0000 1000 0000 01fe 0000 0001 617 * 0000020 0000 1000 0000 01fe 618 * 619 * In this example, compression supports 0x1000 (4,096) data byte length 620 * and 0x1fe (510) total scatter list elements. Decompression supports 621 * 0x1000 (4,096) data byte length and 0x1f3 (510) total scatter list 622 * elements. 623 * 624 * @devdata - struct nx842_devdata to update 625 * @prop - struct property point containing the maxsyncop for the update 626 * 627 * Returns: 628 * 0 on success 629 * -EINVAL on failure 630 */ 631static int nx842_OF_upd_maxsyncop(struct nx842_devdata *devdata, 632 struct property *prop) { 633 int ret = 0; 634 unsigned int comp_data_limit, decomp_data_limit; 635 unsigned int comp_sg_limit, decomp_sg_limit; 636 const struct maxsynccop_t { 637 __be32 comp_elements; 638 __be32 comp_data_limit; 639 __be32 comp_sg_limit; 640 __be32 decomp_elements; 641 __be32 decomp_data_limit; 642 __be32 decomp_sg_limit; 643 } *maxsynccop; 644 645 if (prop->length != sizeof(*maxsynccop)) { 646 dev_err(devdata->dev, "%s: unexpected format for ibm,max-sync-cop property\n", __func__); 647 dev_dbg(devdata->dev, "%s: ibm,max-sync-cop is %d bytes long, expected %lu bytes\n", __func__, prop->length, 648 sizeof(*maxsynccop)); 649 ret = -EINVAL; 650 goto out; 651 } 652 653 maxsynccop = (const struct maxsynccop_t *)prop->value; 654 comp_data_limit = be32_to_cpu(maxsynccop->comp_data_limit); 655 comp_sg_limit = be32_to_cpu(maxsynccop->comp_sg_limit); 656 decomp_data_limit = be32_to_cpu(maxsynccop->decomp_data_limit); 657 decomp_sg_limit = be32_to_cpu(maxsynccop->decomp_sg_limit); 658 659 /* Use one limit rather than separate limits for compression and 660 * decompression. Set a maximum for this so as not to exceed the 661 * size that the header can support and round the value down to 662 * the hardware page size (4K) */ 663 devdata->max_sync_size = min(comp_data_limit, decomp_data_limit); 664 665 devdata->max_sync_size = min_t(unsigned int, devdata->max_sync_size, 666 65536); 667 668 if (devdata->max_sync_size < 4096) { 669 dev_err(devdata->dev, "%s: hardware max data size (%u) is " 670 "less than the driver minimum, unable to use " 671 "the hardware device\n", 672 __func__, devdata->max_sync_size); 673 ret = -EINVAL; 674 goto out; 675 } 676 677 nx842_pseries_constraints.maximum = devdata->max_sync_size; 678 679 devdata->max_sync_sg = min(comp_sg_limit, decomp_sg_limit); 680 if (devdata->max_sync_sg < 1) { 681 dev_err(devdata->dev, "%s: hardware max sg size (%u) is " 682 "less than the driver minimum, unable to use " 683 "the hardware device\n", 684 __func__, devdata->max_sync_sg); 685 ret = -EINVAL; 686 goto out; 687 } 688 689out: 690 return ret; 691} 692 693/** 694 * 695 * nx842_OF_upd -- Handle OF properties updates for the device. 696 * 697 * Set all properties from the OF tree. Optionally, a new property 698 * can be provided by the @new_prop pointer to overwrite an existing value. 699 * The device will remain disabled until all values are valid, this function 700 * will return an error for updates unless all values are valid. 701 * 702 * @new_prop: If not NULL, this property is being updated. If NULL, update 703 * all properties from the current values in the OF tree. 704 * 705 * Returns: 706 * 0 - Success 707 * -ENOMEM - Could not allocate memory for new devdata structure 708 * -EINVAL - property value not found, new_prop is not a recognized 709 * property for the device or property value is not valid. 710 * -ENODEV - Device is not available 711 */ 712static int nx842_OF_upd(struct property *new_prop) 713{ 714 struct nx842_devdata *old_devdata = NULL; 715 struct nx842_devdata *new_devdata = NULL; 716 struct device_node *of_node = NULL; 717 struct property *status = NULL; 718 struct property *maxsglen = NULL; 719 struct property *maxsyncop = NULL; 720 int ret = 0; 721 unsigned long flags; 722 723 new_devdata = kzalloc(sizeof(*new_devdata), GFP_NOFS); 724 if (!new_devdata) 725 return -ENOMEM; 726 727 spin_lock_irqsave(&devdata_mutex, flags); 728 old_devdata = rcu_dereference_check(devdata, 729 lockdep_is_held(&devdata_mutex)); 730 if (old_devdata) 731 of_node = old_devdata->dev->of_node; 732 733 if (!old_devdata || !of_node) { 734 pr_err("%s: device is not available\n", __func__); 735 spin_unlock_irqrestore(&devdata_mutex, flags); 736 kfree(new_devdata); 737 return -ENODEV; 738 } 739 740 memcpy(new_devdata, old_devdata, sizeof(*old_devdata)); 741 new_devdata->counters = old_devdata->counters; 742 743 /* Set ptrs for existing properties */ 744 status = of_find_property(of_node, "status", NULL); 745 maxsglen = of_find_property(of_node, "ibm,max-sg-len", NULL); 746 maxsyncop = of_find_property(of_node, "ibm,max-sync-cop", NULL); 747 if (!status || !maxsglen || !maxsyncop) { 748 dev_err(old_devdata->dev, "%s: Could not locate device properties\n", __func__); 749 ret = -EINVAL; 750 goto error_out; 751 } 752 753 /* 754 * If this is a property update, there are only certain properties that 755 * we care about. Bail if it isn't in the below list 756 */ 757 if (new_prop && (strncmp(new_prop->name, "status", new_prop->length) || 758 strncmp(new_prop->name, "ibm,max-sg-len", new_prop->length) || 759 strncmp(new_prop->name, "ibm,max-sync-cop", new_prop->length))) 760 goto out; 761 762 /* Perform property updates */ 763 ret = nx842_OF_upd_status(new_devdata, status); 764 if (ret) 765 goto error_out; 766 767 ret = nx842_OF_upd_maxsglen(new_devdata, maxsglen); 768 if (ret) 769 goto error_out; 770 771 ret = nx842_OF_upd_maxsyncop(new_devdata, maxsyncop); 772 if (ret) 773 goto error_out; 774 775out: 776 dev_info(old_devdata->dev, "%s: max_sync_size new:%u old:%u\n", 777 __func__, new_devdata->max_sync_size, 778 old_devdata->max_sync_size); 779 dev_info(old_devdata->dev, "%s: max_sync_sg new:%u old:%u\n", 780 __func__, new_devdata->max_sync_sg, 781 old_devdata->max_sync_sg); 782 dev_info(old_devdata->dev, "%s: max_sg_len new:%u old:%u\n", 783 __func__, new_devdata->max_sg_len, 784 old_devdata->max_sg_len); 785 786 rcu_assign_pointer(devdata, new_devdata); 787 spin_unlock_irqrestore(&devdata_mutex, flags); 788 synchronize_rcu(); 789 dev_set_drvdata(new_devdata->dev, new_devdata); 790 kfree(old_devdata); 791 return 0; 792 793error_out: 794 if (new_devdata) { 795 dev_info(old_devdata->dev, "%s: device disabled\n", __func__); 796 nx842_OF_set_defaults(new_devdata); 797 rcu_assign_pointer(devdata, new_devdata); 798 spin_unlock_irqrestore(&devdata_mutex, flags); 799 synchronize_rcu(); 800 dev_set_drvdata(new_devdata->dev, new_devdata); 801 kfree(old_devdata); 802 } else { 803 dev_err(old_devdata->dev, "%s: could not update driver from hardware\n", __func__); 804 spin_unlock_irqrestore(&devdata_mutex, flags); 805 } 806 807 if (!ret) 808 ret = -EINVAL; 809 return ret; 810} 811 812/** 813 * nx842_OF_notifier - Process updates to OF properties for the device 814 * 815 * @np: notifier block 816 * @action: notifier action 817 * @update: struct pSeries_reconfig_prop_update pointer if action is 818 * PSERIES_UPDATE_PROPERTY 819 * 820 * Returns: 821 * NOTIFY_OK on success 822 * NOTIFY_BAD encoded with error number on failure, use 823 * notifier_to_errno() to decode this value 824 */ 825static int nx842_OF_notifier(struct notifier_block *np, unsigned long action, 826 void *data) 827{ 828 struct of_reconfig_data *upd = data; 829 struct nx842_devdata *local_devdata; 830 struct device_node *node = NULL; 831 832 rcu_read_lock(); 833 local_devdata = rcu_dereference(devdata); 834 if (local_devdata) 835 node = local_devdata->dev->of_node; 836 837 if (local_devdata && 838 action == OF_RECONFIG_UPDATE_PROPERTY && 839 !strcmp(upd->dn->name, node->name)) { 840 rcu_read_unlock(); 841 nx842_OF_upd(upd->prop); 842 } else 843 rcu_read_unlock(); 844 845 return NOTIFY_OK; 846} 847 848static struct notifier_block nx842_of_nb = { 849 .notifier_call = nx842_OF_notifier, 850}; 851 852#define nx842_counter_read(_name) \ 853static ssize_t nx842_##_name##_show(struct device *dev, \ 854 struct device_attribute *attr, \ 855 char *buf) { \ 856 struct nx842_devdata *local_devdata; \ 857 int p = 0; \ 858 rcu_read_lock(); \ 859 local_devdata = rcu_dereference(devdata); \ 860 if (local_devdata) \ 861 p = snprintf(buf, PAGE_SIZE, "%lld\n", \ 862 atomic64_read(&local_devdata->counters->_name)); \ 863 rcu_read_unlock(); \ 864 return p; \ 865} 866 867#define NX842DEV_COUNTER_ATTR_RO(_name) \ 868 nx842_counter_read(_name); \ 869 static struct device_attribute dev_attr_##_name = __ATTR(_name, \ 870 0444, \ 871 nx842_##_name##_show,\ 872 NULL); 873 874NX842DEV_COUNTER_ATTR_RO(comp_complete); 875NX842DEV_COUNTER_ATTR_RO(comp_failed); 876NX842DEV_COUNTER_ATTR_RO(decomp_complete); 877NX842DEV_COUNTER_ATTR_RO(decomp_failed); 878NX842DEV_COUNTER_ATTR_RO(swdecomp); 879 880static ssize_t nx842_timehist_show(struct device *, 881 struct device_attribute *, char *); 882 883static struct device_attribute dev_attr_comp_times = __ATTR(comp_times, 0444, 884 nx842_timehist_show, NULL); 885static struct device_attribute dev_attr_decomp_times = __ATTR(decomp_times, 886 0444, nx842_timehist_show, NULL); 887 888static ssize_t nx842_timehist_show(struct device *dev, 889 struct device_attribute *attr, char *buf) { 890 char *p = buf; 891 struct nx842_devdata *local_devdata; 892 atomic64_t *times; 893 int bytes_remain = PAGE_SIZE; 894 int bytes; 895 int i; 896 897 rcu_read_lock(); 898 local_devdata = rcu_dereference(devdata); 899 if (!local_devdata) { 900 rcu_read_unlock(); 901 return 0; 902 } 903 904 if (attr == &dev_attr_comp_times) 905 times = local_devdata->counters->comp_times; 906 else if (attr == &dev_attr_decomp_times) 907 times = local_devdata->counters->decomp_times; 908 else { 909 rcu_read_unlock(); 910 return 0; 911 } 912 913 for (i = 0; i < (NX842_HIST_SLOTS - 2); i++) { 914 bytes = snprintf(p, bytes_remain, "%u-%uus:\t%lld\n", 915 i ? (2<<(i-1)) : 0, (2<<i)-1, 916 atomic64_read(×[i])); 917 bytes_remain -= bytes; 918 p += bytes; 919 } 920 /* The last bucket holds everything over 921 * 2<<(NX842_HIST_SLOTS - 2) us */ 922 bytes = snprintf(p, bytes_remain, "%uus - :\t%lld\n", 923 2<<(NX842_HIST_SLOTS - 2), 924 atomic64_read(×[(NX842_HIST_SLOTS - 1)])); 925 p += bytes; 926 927 rcu_read_unlock(); 928 return p - buf; 929} 930 931static struct attribute *nx842_sysfs_entries[] = { 932 &dev_attr_comp_complete.attr, 933 &dev_attr_comp_failed.attr, 934 &dev_attr_decomp_complete.attr, 935 &dev_attr_decomp_failed.attr, 936 &dev_attr_swdecomp.attr, 937 &dev_attr_comp_times.attr, 938 &dev_attr_decomp_times.attr, 939 NULL, 940}; 941 942static struct attribute_group nx842_attribute_group = { 943 .name = NULL, /* put in device directory */ 944 .attrs = nx842_sysfs_entries, 945}; 946 947static struct nx842_driver nx842_pseries_driver = { 948 .name = KBUILD_MODNAME, 949 .owner = THIS_MODULE, 950 .workmem_size = sizeof(struct nx842_workmem), 951 .constraints = &nx842_pseries_constraints, 952 .compress = nx842_pseries_compress, 953 .decompress = nx842_pseries_decompress, 954}; 955 956static int nx842_pseries_crypto_init(struct crypto_tfm *tfm) 957{ 958 return nx842_crypto_init(tfm, &nx842_pseries_driver); 959} 960 961static struct crypto_alg nx842_pseries_alg = { 962 .cra_name = "842", 963 .cra_driver_name = "842-nx", 964 .cra_priority = 300, 965 .cra_flags = CRYPTO_ALG_TYPE_COMPRESS, 966 .cra_ctxsize = sizeof(struct nx842_crypto_ctx), 967 .cra_module = THIS_MODULE, 968 .cra_init = nx842_pseries_crypto_init, 969 .cra_exit = nx842_crypto_exit, 970 .cra_u = { .compress = { 971 .coa_compress = nx842_crypto_compress, 972 .coa_decompress = nx842_crypto_decompress } } 973}; 974 975static int nx842_probe(struct vio_dev *viodev, 976 const struct vio_device_id *id) 977{ 978 struct nx842_devdata *old_devdata, *new_devdata = NULL; 979 unsigned long flags; 980 int ret = 0; 981 982 new_devdata = kzalloc(sizeof(*new_devdata), GFP_NOFS); 983 if (!new_devdata) 984 return -ENOMEM; 985 986 new_devdata->counters = kzalloc(sizeof(*new_devdata->counters), 987 GFP_NOFS); 988 if (!new_devdata->counters) { 989 kfree(new_devdata); 990 return -ENOMEM; 991 } 992 993 spin_lock_irqsave(&devdata_mutex, flags); 994 old_devdata = rcu_dereference_check(devdata, 995 lockdep_is_held(&devdata_mutex)); 996 997 if (old_devdata && old_devdata->vdev != NULL) { 998 dev_err(&viodev->dev, "%s: Attempt to register more than one instance of the hardware\n", __func__); 999 ret = -1; 1000 goto error_unlock; 1001 } 1002 1003 dev_set_drvdata(&viodev->dev, NULL); 1004 1005 new_devdata->vdev = viodev; 1006 new_devdata->dev = &viodev->dev; 1007 nx842_OF_set_defaults(new_devdata); 1008 1009 rcu_assign_pointer(devdata, new_devdata); 1010 spin_unlock_irqrestore(&devdata_mutex, flags); 1011 synchronize_rcu(); 1012 kfree(old_devdata); 1013 1014 of_reconfig_notifier_register(&nx842_of_nb); 1015 1016 ret = nx842_OF_upd(NULL); 1017 if (ret) 1018 goto error; 1019 1020 ret = crypto_register_alg(&nx842_pseries_alg); 1021 if (ret) { 1022 dev_err(&viodev->dev, "could not register comp alg: %d\n", ret); 1023 goto error; 1024 } 1025 1026 rcu_read_lock(); 1027 dev_set_drvdata(&viodev->dev, rcu_dereference(devdata)); 1028 rcu_read_unlock(); 1029 1030 if (sysfs_create_group(&viodev->dev.kobj, &nx842_attribute_group)) { 1031 dev_err(&viodev->dev, "could not create sysfs device attributes\n"); 1032 ret = -1; 1033 goto error; 1034 } 1035 1036 return 0; 1037 1038error_unlock: 1039 spin_unlock_irqrestore(&devdata_mutex, flags); 1040 if (new_devdata) 1041 kfree(new_devdata->counters); 1042 kfree(new_devdata); 1043error: 1044 return ret; 1045} 1046 1047static int nx842_remove(struct vio_dev *viodev) 1048{ 1049 struct nx842_devdata *old_devdata; 1050 unsigned long flags; 1051 1052 pr_info("Removing IBM Power 842 compression device\n"); 1053 sysfs_remove_group(&viodev->dev.kobj, &nx842_attribute_group); 1054 1055 crypto_unregister_alg(&nx842_pseries_alg); 1056 1057 spin_lock_irqsave(&devdata_mutex, flags); 1058 old_devdata = rcu_dereference_check(devdata, 1059 lockdep_is_held(&devdata_mutex)); 1060 of_reconfig_notifier_unregister(&nx842_of_nb); 1061 RCU_INIT_POINTER(devdata, NULL); 1062 spin_unlock_irqrestore(&devdata_mutex, flags); 1063 synchronize_rcu(); 1064 dev_set_drvdata(&viodev->dev, NULL); 1065 if (old_devdata) 1066 kfree(old_devdata->counters); 1067 kfree(old_devdata); 1068 1069 return 0; 1070} 1071 1072static const struct vio_device_id nx842_vio_driver_ids[] = { 1073 {"ibm,compression-v1", "ibm,compression"}, 1074 {"", ""}, 1075}; 1076MODULE_DEVICE_TABLE(vio, nx842_vio_driver_ids); 1077 1078static struct vio_driver nx842_vio_driver = { 1079 .name = KBUILD_MODNAME, 1080 .probe = nx842_probe, 1081 .remove = nx842_remove, 1082 .get_desired_dma = nx842_get_desired_dma, 1083 .id_table = nx842_vio_driver_ids, 1084}; 1085 1086static int __init nx842_pseries_init(void) 1087{ 1088 struct nx842_devdata *new_devdata; 1089 int ret; 1090 1091 if (!of_find_compatible_node(NULL, NULL, "ibm,compression")) 1092 return -ENODEV; 1093 1094 RCU_INIT_POINTER(devdata, NULL); 1095 new_devdata = kzalloc(sizeof(*new_devdata), GFP_KERNEL); 1096 if (!new_devdata) 1097 return -ENOMEM; 1098 1099 RCU_INIT_POINTER(devdata, new_devdata); 1100 1101 ret = vio_register_driver(&nx842_vio_driver); 1102 if (ret) { 1103 pr_err("Could not register VIO driver %d\n", ret); 1104 1105 kfree(new_devdata); 1106 return ret; 1107 } 1108 1109 return 0; 1110} 1111 1112module_init(nx842_pseries_init); 1113 1114static void __exit nx842_pseries_exit(void) 1115{ 1116 struct nx842_devdata *old_devdata; 1117 unsigned long flags; 1118 1119 crypto_unregister_alg(&nx842_pseries_alg); 1120 1121 spin_lock_irqsave(&devdata_mutex, flags); 1122 old_devdata = rcu_dereference_check(devdata, 1123 lockdep_is_held(&devdata_mutex)); 1124 RCU_INIT_POINTER(devdata, NULL); 1125 spin_unlock_irqrestore(&devdata_mutex, flags); 1126 synchronize_rcu(); 1127 if (old_devdata && old_devdata->dev) 1128 dev_set_drvdata(old_devdata->dev, NULL); 1129 kfree(old_devdata); 1130 vio_unregister_driver(&nx842_vio_driver); 1131} 1132 1133module_exit(nx842_pseries_exit); 1134 1135