1/******************************************************************************* 2* 3* Copyright (c) 2015-2016 Intel Corporation. All rights reserved. 4* 5* This software is available to you under a choice of one of two 6* licenses. You may choose to be licensed under the terms of the GNU 7* General Public License (GPL) Version 2, available from the file 8* COPYING in the main directory of this source tree, or the 9* OpenFabrics.org BSD license below: 10* 11* Redistribution and use in source and binary forms, with or 12* without modification, are permitted provided that the following 13* conditions are met: 14* 15* - Redistributions of source code must retain the above 16* copyright notice, this list of conditions and the following 17* disclaimer. 18* 19* - Redistributions in binary form must reproduce the above 20* copyright notice, this list of conditions and the following 21* disclaimer in the documentation and/or other materials 22* provided with the distribution. 23* 24* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31* SOFTWARE. 32* 33*******************************************************************************/ 34 35#ifndef I40IW_IW_H 36#define I40IW_IW_H 37#include <linux/netdevice.h> 38#include <linux/inetdevice.h> 39#include <linux/spinlock.h> 40#include <linux/kernel.h> 41#include <linux/delay.h> 42#include <linux/pci.h> 43#include <linux/dma-mapping.h> 44#include <linux/workqueue.h> 45#include <linux/slab.h> 46#include <linux/io.h> 47#include <linux/crc32c.h> 48#include <linux/net/intel/i40e_client.h> 49#include <rdma/ib_smi.h> 50#include <rdma/ib_verbs.h> 51#include <rdma/ib_pack.h> 52#include <rdma/rdma_cm.h> 53#include <rdma/iw_cm.h> 54#include <crypto/hash.h> 55 56#include "i40iw_status.h" 57#include "i40iw_osdep.h" 58#include "i40iw_d.h" 59#include "i40iw_hmc.h" 60 61#include "i40iw_type.h" 62#include "i40iw_p.h" 63#include <rdma/i40iw-abi.h> 64#include "i40iw_pble.h" 65#include "i40iw_verbs.h" 66#include "i40iw_cm.h" 67#include "i40iw_user.h" 68#include "i40iw_puda.h" 69 70#define I40IW_FW_VER_DEFAULT 2 71#define I40IW_HW_VERSION 2 72 73#define I40IW_ARP_ADD 1 74#define I40IW_ARP_DELETE 2 75#define I40IW_ARP_RESOLVE 3 76 77#define I40IW_MACIP_ADD 1 78#define I40IW_MACIP_DELETE 2 79 80#define IW_CCQ_SIZE (I40IW_CQP_SW_SQSIZE_2048 + 1) 81#define IW_CEQ_SIZE 2048 82#define IW_AEQ_SIZE 2048 83 84#define RX_BUF_SIZE (1536 + 8) 85#define IW_REG0_SIZE (4 * 1024) 86#define IW_TX_TIMEOUT (6 * HZ) 87#define IW_FIRST_QPN 1 88#define IW_SW_CONTEXT_ALIGN 1024 89 90#define MAX_DPC_ITERATIONS 128 91 92#define I40IW_EVENT_TIMEOUT 100000 93#define I40IW_VCHNL_EVENT_TIMEOUT 100000 94 95#define I40IW_NO_VLAN 0xffff 96#define I40IW_NO_QSET 0xffff 97 98/* access to mcast filter list */ 99#define IW_ADD_MCAST false 100#define IW_DEL_MCAST true 101 102#define I40IW_DRV_OPT_ENABLE_MPA_VER_0 0x00000001 103#define I40IW_DRV_OPT_DISABLE_MPA_CRC 0x00000002 104#define I40IW_DRV_OPT_DISABLE_FIRST_WRITE 0x00000004 105#define I40IW_DRV_OPT_DISABLE_INTF 0x00000008 106#define I40IW_DRV_OPT_ENABLE_MSI 0x00000010 107#define I40IW_DRV_OPT_DUAL_LOGICAL_PORT 0x00000020 108#define I40IW_DRV_OPT_NO_INLINE_DATA 0x00000080 109#define I40IW_DRV_OPT_DISABLE_INT_MOD 0x00000100 110#define I40IW_DRV_OPT_DISABLE_VIRT_WQ 0x00000200 111#define I40IW_DRV_OPT_ENABLE_PAU 0x00000400 112#define I40IW_DRV_OPT_MCAST_LOGPORT_MAP 0x00000800 113 114#define IW_HMC_OBJ_TYPE_NUM ARRAY_SIZE(iw_hmc_obj_types) 115#define IW_CFG_FPM_QP_COUNT 32768 116#define I40IW_MAX_PAGES_PER_FMR 512 117#define I40IW_MIN_PAGES_PER_FMR 1 118#define I40IW_CQP_COMPL_RQ_WQE_FLUSHED 2 119#define I40IW_CQP_COMPL_SQ_WQE_FLUSHED 3 120#define I40IW_CQP_COMPL_RQ_SQ_WQE_FLUSHED 4 121 122struct i40iw_cqp_compl_info { 123 u32 op_ret_val; 124 u16 maj_err_code; 125 u16 min_err_code; 126 bool error; 127 u8 op_code; 128}; 129 130#define i40iw_pr_err(fmt, args ...) pr_err("%s: "fmt, __func__, ## args) 131 132#define i40iw_pr_info(fmt, args ...) pr_info("%s: " fmt, __func__, ## args) 133 134#define i40iw_pr_warn(fmt, args ...) pr_warn("%s: " fmt, __func__, ## args) 135 136struct i40iw_cqp_request { 137 struct cqp_commands_info info; 138 wait_queue_head_t waitq; 139 struct list_head list; 140 atomic_t refcount; 141 void (*callback_fcn)(struct i40iw_cqp_request*, u32); 142 void *param; 143 struct i40iw_cqp_compl_info compl_info; 144 bool waiting; 145 bool request_done; 146 bool dynamic; 147}; 148 149struct i40iw_cqp { 150 struct i40iw_sc_cqp sc_cqp; 151 spinlock_t req_lock; /*cqp request list */ 152 wait_queue_head_t waitq; 153 struct i40iw_dma_mem sq; 154 struct i40iw_dma_mem host_ctx; 155 u64 *scratch_array; 156 struct i40iw_cqp_request *cqp_requests; 157 struct list_head cqp_avail_reqs; 158 struct list_head cqp_pending_reqs; 159}; 160 161struct i40iw_device; 162 163struct i40iw_ccq { 164 struct i40iw_sc_cq sc_cq; 165 spinlock_t lock; /* ccq control */ 166 wait_queue_head_t waitq; 167 struct i40iw_dma_mem mem_cq; 168 struct i40iw_dma_mem shadow_area; 169}; 170 171struct i40iw_ceq { 172 struct i40iw_sc_ceq sc_ceq; 173 struct i40iw_dma_mem mem; 174 u32 irq; 175 u32 msix_idx; 176 struct i40iw_device *iwdev; 177 struct tasklet_struct dpc_tasklet; 178}; 179 180struct i40iw_aeq { 181 struct i40iw_sc_aeq sc_aeq; 182 struct i40iw_dma_mem mem; 183}; 184 185struct i40iw_arp_entry { 186 u32 ip_addr[4]; 187 u8 mac_addr[ETH_ALEN]; 188}; 189 190enum init_completion_state { 191 INVALID_STATE = 0, 192 INITIAL_STATE, 193 CQP_CREATED, 194 HMC_OBJS_CREATED, 195 PBLE_CHUNK_MEM, 196 CCQ_CREATED, 197 AEQ_CREATED, 198 CEQ_CREATED, 199 ILQ_CREATED, 200 IEQ_CREATED, 201 IP_ADDR_REGISTERED, 202 RDMA_DEV_REGISTERED 203}; 204 205struct i40iw_msix_vector { 206 u32 idx; 207 u32 irq; 208 u32 cpu_affinity; 209 u32 ceq_id; 210 cpumask_t mask; 211}; 212 213struct l2params_work { 214 struct work_struct work; 215 struct i40iw_device *iwdev; 216 struct i40iw_l2params l2params; 217}; 218 219#define I40IW_MSIX_TABLE_SIZE 65 220 221struct virtchnl_work { 222 struct work_struct work; 223 union { 224 struct i40iw_cqp_request *cqp_request; 225 struct i40iw_virtchnl_work_info work_info; 226 }; 227}; 228 229struct i40e_qvlist_info; 230 231struct i40iw_device { 232 struct i40iw_ib_device *iwibdev; 233 struct net_device *netdev; 234 wait_queue_head_t vchnl_waitq; 235 struct i40iw_sc_dev sc_dev; 236 struct i40iw_sc_vsi vsi; 237 struct i40iw_handler *hdl; 238 struct i40e_info *ldev; 239 struct i40e_client *client; 240 struct i40iw_hw hw; 241 struct i40iw_cm_core cm_core; 242 u8 *mem_resources; 243 unsigned long *allocated_qps; 244 unsigned long *allocated_cqs; 245 unsigned long *allocated_mrs; 246 unsigned long *allocated_pds; 247 unsigned long *allocated_arps; 248 struct i40iw_qp **qp_table; 249 bool msix_shared; 250 u32 msix_count; 251 struct i40iw_msix_vector *iw_msixtbl; 252 struct i40e_qvlist_info *iw_qvlist; 253 254 struct i40iw_hmc_pble_rsrc *pble_rsrc; 255 struct i40iw_arp_entry *arp_table; 256 struct i40iw_cqp cqp; 257 struct i40iw_ccq ccq; 258 u32 ceqs_count; 259 struct i40iw_ceq *ceqlist; 260 struct i40iw_aeq aeq; 261 u32 arp_table_size; 262 u32 next_arp_index; 263 spinlock_t resource_lock; /* hw resource access */ 264 spinlock_t qptable_lock; 265 u32 vendor_id; 266 u32 vendor_part_id; 267 u32 of_device_registered; 268 269 u32 device_cap_flags; 270 unsigned long db_start; 271 u8 resource_profile; 272 u8 max_rdma_vfs; 273 u8 max_enabled_vfs; 274 u8 max_sge; 275 u8 iw_status; 276 u8 send_term_ok; 277 bool push_mode; /* Initialized from parameter passed to driver */ 278 279 /* x710 specific */ 280 struct mutex pbl_mutex; 281 struct tasklet_struct dpc_tasklet; 282 struct workqueue_struct *virtchnl_wq; 283 struct virtchnl_work virtchnl_w[I40IW_MAX_PE_ENABLED_VF_COUNT]; 284 struct i40iw_dma_mem obj_mem; 285 struct i40iw_dma_mem obj_next; 286 u8 *hmc_info_mem; 287 u32 sd_type; 288 struct workqueue_struct *param_wq; 289 atomic_t params_busy; 290 enum init_completion_state init_state; 291 u16 mac_ip_table_idx; 292 atomic_t vchnl_msgs; 293 u32 max_mr; 294 u32 max_qp; 295 u32 max_cq; 296 u32 max_pd; 297 u32 next_qp; 298 u32 next_cq; 299 u32 next_pd; 300 u32 max_mr_size; 301 u32 max_qp_wr; 302 u32 max_cqe; 303 u32 mr_stagmask; 304 u32 mpa_version; 305 bool dcb; 306 bool closing; 307 bool reset; 308 u32 used_pds; 309 u32 used_cqs; 310 u32 used_mrs; 311 u32 used_qps; 312 wait_queue_head_t close_wq; 313 atomic64_t use_count; 314}; 315 316struct i40iw_ib_device { 317 struct ib_device ibdev; 318 struct i40iw_device *iwdev; 319}; 320 321struct i40iw_handler { 322 struct list_head list; 323 struct i40e_client *client; 324 struct i40iw_device device; 325 struct i40e_info ldev; 326}; 327 328/** 329 * i40iw_fw_major_ver - get firmware major version 330 * @dev: iwarp device 331 **/ 332static inline u64 i40iw_fw_major_ver(struct i40iw_sc_dev *dev) 333{ 334 return RS_64(dev->feature_info[I40IW_FEATURE_FW_INFO], 335 I40IW_FW_VER_MAJOR); 336} 337 338/** 339 * i40iw_fw_minor_ver - get firmware minor version 340 * @dev: iwarp device 341 **/ 342static inline u64 i40iw_fw_minor_ver(struct i40iw_sc_dev *dev) 343{ 344 return RS_64(dev->feature_info[I40IW_FEATURE_FW_INFO], 345 I40IW_FW_VER_MINOR); 346} 347 348/** 349 * to_iwdev - get device 350 * @ibdev: ib device 351 **/ 352static inline struct i40iw_device *to_iwdev(struct ib_device *ibdev) 353{ 354 return container_of(ibdev, struct i40iw_ib_device, ibdev)->iwdev; 355} 356 357/** 358 * to_ucontext - get user context 359 * @ibucontext: ib user context 360 **/ 361static inline struct i40iw_ucontext *to_ucontext(struct ib_ucontext *ibucontext) 362{ 363 return container_of(ibucontext, struct i40iw_ucontext, ibucontext); 364} 365 366/** 367 * to_iwpd - get protection domain 368 * @ibpd: ib pd 369 **/ 370static inline struct i40iw_pd *to_iwpd(struct ib_pd *ibpd) 371{ 372 return container_of(ibpd, struct i40iw_pd, ibpd); 373} 374 375/** 376 * to_iwmr - get device memory region 377 * @ibdev: ib memory region 378 **/ 379static inline struct i40iw_mr *to_iwmr(struct ib_mr *ibmr) 380{ 381 return container_of(ibmr, struct i40iw_mr, ibmr); 382} 383 384/** 385 * to_iwmw - get device memory window 386 * @ibmw: ib memory window 387 **/ 388static inline struct i40iw_mr *to_iwmw(struct ib_mw *ibmw) 389{ 390 return container_of(ibmw, struct i40iw_mr, ibmw); 391} 392 393/** 394 * to_iwcq - get completion queue 395 * @ibcq: ib cqdevice 396 **/ 397static inline struct i40iw_cq *to_iwcq(struct ib_cq *ibcq) 398{ 399 return container_of(ibcq, struct i40iw_cq, ibcq); 400} 401 402/** 403 * to_iwqp - get device qp 404 * @ibqp: ib qp 405 **/ 406static inline struct i40iw_qp *to_iwqp(struct ib_qp *ibqp) 407{ 408 return container_of(ibqp, struct i40iw_qp, ibqp); 409} 410 411/* i40iw.c */ 412void i40iw_qp_add_ref(struct ib_qp *ibqp); 413void i40iw_qp_rem_ref(struct ib_qp *ibqp); 414struct ib_qp *i40iw_get_qp(struct ib_device *, int); 415 416void i40iw_flush_wqes(struct i40iw_device *iwdev, 417 struct i40iw_qp *qp); 418 419void i40iw_manage_arp_cache(struct i40iw_device *iwdev, 420 unsigned char *mac_addr, 421 u32 *ip_addr, 422 bool ipv4, 423 u32 action); 424 425enum i40iw_status_code i40iw_manage_apbvt(struct i40iw_device *iwdev, 426 u16 accel_local_port, bool add_port); 427 428struct i40iw_cqp_request *i40iw_get_cqp_request(struct i40iw_cqp *cqp, bool wait); 429void i40iw_free_cqp_request(struct i40iw_cqp *cqp, struct i40iw_cqp_request *cqp_request); 430void i40iw_put_cqp_request(struct i40iw_cqp *cqp, struct i40iw_cqp_request *cqp_request); 431 432/** 433 * i40iw_alloc_resource - allocate a resource 434 * @iwdev: device pointer 435 * @resource_array: resource bit array: 436 * @max_resources: maximum resource number 437 * @req_resources_num: Allocated resource number 438 * @next: next free id 439 **/ 440static inline int i40iw_alloc_resource(struct i40iw_device *iwdev, 441 unsigned long *resource_array, 442 u32 max_resources, 443 u32 *req_resource_num, 444 u32 *next) 445{ 446 u32 resource_num; 447 unsigned long flags; 448 449 spin_lock_irqsave(&iwdev->resource_lock, flags); 450 resource_num = find_next_zero_bit(resource_array, max_resources, *next); 451 if (resource_num >= max_resources) { 452 resource_num = find_first_zero_bit(resource_array, max_resources); 453 if (resource_num >= max_resources) { 454 spin_unlock_irqrestore(&iwdev->resource_lock, flags); 455 return -EOVERFLOW; 456 } 457 } 458 set_bit(resource_num, resource_array); 459 *next = resource_num + 1; 460 if (*next == max_resources) 461 *next = 0; 462 *req_resource_num = resource_num; 463 spin_unlock_irqrestore(&iwdev->resource_lock, flags); 464 465 return 0; 466} 467 468/** 469 * i40iw_is_resource_allocated - detrmine if resource is 470 * allocated 471 * @iwdev: device pointer 472 * @resource_array: resource array for the resource_num 473 * @resource_num: resource number to check 474 **/ 475static inline bool i40iw_is_resource_allocated(struct i40iw_device *iwdev, 476 unsigned long *resource_array, 477 u32 resource_num) 478{ 479 bool bit_is_set; 480 unsigned long flags; 481 482 spin_lock_irqsave(&iwdev->resource_lock, flags); 483 484 bit_is_set = test_bit(resource_num, resource_array); 485 spin_unlock_irqrestore(&iwdev->resource_lock, flags); 486 487 return bit_is_set; 488} 489 490/** 491 * i40iw_free_resource - free a resource 492 * @iwdev: device pointer 493 * @resource_array: resource array for the resource_num 494 * @resource_num: resource number to free 495 **/ 496static inline void i40iw_free_resource(struct i40iw_device *iwdev, 497 unsigned long *resource_array, 498 u32 resource_num) 499{ 500 unsigned long flags; 501 502 spin_lock_irqsave(&iwdev->resource_lock, flags); 503 clear_bit(resource_num, resource_array); 504 spin_unlock_irqrestore(&iwdev->resource_lock, flags); 505} 506 507/** 508 * to_iwhdl - Get the handler from the device pointer 509 * @iwdev: device pointer 510 **/ 511static inline struct i40iw_handler *to_iwhdl(struct i40iw_device *iw_dev) 512{ 513 return container_of(iw_dev, struct i40iw_handler, device); 514} 515 516struct i40iw_handler *i40iw_find_netdev(struct net_device *netdev); 517 518/** 519 * iw_init_resources - 520 */ 521u32 i40iw_initialize_hw_resources(struct i40iw_device *iwdev); 522 523int i40iw_register_rdma_device(struct i40iw_device *iwdev); 524void i40iw_port_ibevent(struct i40iw_device *iwdev); 525void i40iw_cm_disconn(struct i40iw_qp *iwqp); 526void i40iw_cm_disconn_worker(void *); 527int mini_cm_recv_pkt(struct i40iw_cm_core *, struct i40iw_device *, 528 struct sk_buff *); 529 530enum i40iw_status_code i40iw_handle_cqp_op(struct i40iw_device *iwdev, 531 struct i40iw_cqp_request *cqp_request); 532enum i40iw_status_code i40iw_add_mac_addr(struct i40iw_device *iwdev, 533 u8 *mac_addr, u8 *mac_index); 534int i40iw_modify_qp(struct ib_qp *, struct ib_qp_attr *, int, struct ib_udata *); 535void i40iw_cq_wq_destroy(struct i40iw_device *iwdev, struct i40iw_sc_cq *cq); 536 537void i40iw_cleanup_pending_cqp_op(struct i40iw_device *iwdev); 538void i40iw_rem_pdusecount(struct i40iw_pd *iwpd, struct i40iw_device *iwdev); 539void i40iw_add_pdusecount(struct i40iw_pd *iwpd); 540void i40iw_rem_devusecount(struct i40iw_device *iwdev); 541void i40iw_add_devusecount(struct i40iw_device *iwdev); 542void i40iw_hw_modify_qp(struct i40iw_device *iwdev, struct i40iw_qp *iwqp, 543 struct i40iw_modify_qp_info *info, bool wait); 544 545void i40iw_qp_suspend_resume(struct i40iw_sc_dev *dev, 546 struct i40iw_sc_qp *qp, 547 bool suspend); 548enum i40iw_status_code i40iw_manage_qhash(struct i40iw_device *iwdev, 549 struct i40iw_cm_info *cminfo, 550 enum i40iw_quad_entry_type etype, 551 enum i40iw_quad_hash_manage_type mtype, 552 void *cmnode, 553 bool wait); 554void i40iw_receive_ilq(struct i40iw_sc_vsi *vsi, struct i40iw_puda_buf *rbuf); 555void i40iw_free_sqbuf(struct i40iw_sc_vsi *vsi, void *bufp); 556void i40iw_free_qp_resources(struct i40iw_qp *iwqp); 557 558enum i40iw_status_code i40iw_obj_aligned_mem(struct i40iw_device *iwdev, 559 struct i40iw_dma_mem *memptr, 560 u32 size, u32 mask); 561 562void i40iw_request_reset(struct i40iw_device *iwdev); 563void i40iw_destroy_rdma_device(struct i40iw_ib_device *iwibdev); 564int i40iw_setup_cm_core(struct i40iw_device *iwdev); 565void i40iw_cleanup_cm_core(struct i40iw_cm_core *cm_core); 566void i40iw_process_ceq(struct i40iw_device *, struct i40iw_ceq *iwceq); 567void i40iw_process_aeq(struct i40iw_device *); 568void i40iw_next_iw_state(struct i40iw_qp *iwqp, 569 u8 state, u8 del_hash, 570 u8 term, u8 term_len); 571int i40iw_send_syn(struct i40iw_cm_node *cm_node, u32 sendack); 572int i40iw_send_reset(struct i40iw_cm_node *cm_node); 573struct i40iw_cm_node *i40iw_find_node(struct i40iw_cm_core *cm_core, 574 u16 rem_port, 575 u32 *rem_addr, 576 u16 loc_port, 577 u32 *loc_addr, 578 bool add_refcnt, 579 bool accelerated_list); 580 581enum i40iw_status_code i40iw_hw_flush_wqes(struct i40iw_device *iwdev, 582 struct i40iw_sc_qp *qp, 583 struct i40iw_qp_flush_info *info, 584 bool wait); 585 586void i40iw_gen_ae(struct i40iw_device *iwdev, 587 struct i40iw_sc_qp *qp, 588 struct i40iw_gen_ae_info *info, 589 bool wait); 590 591void i40iw_copy_ip_ntohl(u32 *dst, __be32 *src); 592struct ib_mr *i40iw_reg_phys_mr(struct ib_pd *ib_pd, 593 u64 addr, 594 u64 size, 595 int acc, 596 u64 *iova_start); 597 598int i40iw_inetaddr_event(struct notifier_block *notifier, 599 unsigned long event, 600 void *ptr); 601int i40iw_inet6addr_event(struct notifier_block *notifier, 602 unsigned long event, 603 void *ptr); 604int i40iw_net_event(struct notifier_block *notifier, 605 unsigned long event, 606 void *ptr); 607int i40iw_netdevice_event(struct notifier_block *notifier, 608 unsigned long event, 609 void *ptr); 610 611#endif 612