1/* 2 * cxgb4i.c: Chelsio T4 iSCSI driver. 3 * 4 * Copyright (c) 2010-2015 Chelsio Communications, Inc. 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License as published by 8 * the Free Software Foundation. 9 * 10 * Written by: Karen Xie (kxie@chelsio.com) 11 * Rakesh Ranjan (rranjan@chelsio.com) 12 */ 13 14#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__ 15 16#include <linux/kernel.h> 17#include <linux/module.h> 18#include <linux/moduleparam.h> 19#include <scsi/scsi_host.h> 20#include <net/tcp.h> 21#include <net/dst.h> 22#include <linux/netdevice.h> 23#include <net/addrconf.h> 24 25#include "t4_regs.h" 26#include "t4_msg.h" 27#include "cxgb4.h" 28#include "cxgb4_uld.h" 29#include "t4fw_api.h" 30#include "l2t.h" 31#include "cxgb4i.h" 32#include "clip_tbl.h" 33 34static unsigned int dbg_level; 35 36#include "../libcxgbi.h" 37 38#ifdef CONFIG_CHELSIO_T4_DCB 39#include <net/dcbevent.h> 40#include "cxgb4_dcb.h" 41#endif 42 43#define DRV_MODULE_NAME "cxgb4i" 44#define DRV_MODULE_DESC "Chelsio T4-T6 iSCSI Driver" 45#define DRV_MODULE_VERSION "0.9.5-ko" 46#define DRV_MODULE_RELDATE "Apr. 2015" 47 48static char version[] = 49 DRV_MODULE_DESC " " DRV_MODULE_NAME 50 " v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n"; 51 52MODULE_AUTHOR("Chelsio Communications, Inc."); 53MODULE_DESCRIPTION(DRV_MODULE_DESC); 54MODULE_VERSION(DRV_MODULE_VERSION); 55MODULE_LICENSE("GPL"); 56 57module_param(dbg_level, uint, 0644); 58MODULE_PARM_DESC(dbg_level, "Debug flag (default=0)"); 59 60#define CXGB4I_DEFAULT_10G_RCV_WIN (256 * 1024) 61static int cxgb4i_rcv_win = -1; 62module_param(cxgb4i_rcv_win, int, 0644); 63MODULE_PARM_DESC(cxgb4i_rcv_win, "TCP receive window in bytes"); 64 65#define CXGB4I_DEFAULT_10G_SND_WIN (128 * 1024) 66static int cxgb4i_snd_win = -1; 67module_param(cxgb4i_snd_win, int, 0644); 68MODULE_PARM_DESC(cxgb4i_snd_win, "TCP send window in bytes"); 69 70static int cxgb4i_rx_credit_thres = 10 * 1024; 71module_param(cxgb4i_rx_credit_thres, int, 0644); 72MODULE_PARM_DESC(cxgb4i_rx_credit_thres, 73 "RX credits return threshold in bytes (default=10KB)"); 74 75static unsigned int cxgb4i_max_connect = (8 * 1024); 76module_param(cxgb4i_max_connect, uint, 0644); 77MODULE_PARM_DESC(cxgb4i_max_connect, "Maximum number of connections"); 78 79static unsigned short cxgb4i_sport_base = 20000; 80module_param(cxgb4i_sport_base, ushort, 0644); 81MODULE_PARM_DESC(cxgb4i_sport_base, "Starting port number (default 20000)"); 82 83typedef void (*cxgb4i_cplhandler_func)(struct cxgbi_device *, struct sk_buff *); 84 85static void *t4_uld_add(const struct cxgb4_lld_info *); 86static int t4_uld_rx_handler(void *, const __be64 *, const struct pkt_gl *); 87static int t4_uld_state_change(void *, enum cxgb4_state state); 88static inline int send_tx_flowc_wr(struct cxgbi_sock *); 89 90static const struct cxgb4_uld_info cxgb4i_uld_info = { 91 .name = DRV_MODULE_NAME, 92 .nrxq = MAX_ULD_QSETS, 93 .ntxq = MAX_ULD_QSETS, 94 .rxq_size = 1024, 95 .lro = false, 96 .add = t4_uld_add, 97 .rx_handler = t4_uld_rx_handler, 98 .state_change = t4_uld_state_change, 99}; 100 101static struct scsi_host_template cxgb4i_host_template = { 102 .module = THIS_MODULE, 103 .name = DRV_MODULE_NAME, 104 .proc_name = DRV_MODULE_NAME, 105 .can_queue = CXGB4I_SCSI_HOST_QDEPTH, 106 .queuecommand = iscsi_queuecommand, 107 .change_queue_depth = scsi_change_queue_depth, 108 .sg_tablesize = SG_ALL, 109 .max_sectors = 0xFFFF, 110 .cmd_per_lun = ISCSI_DEF_CMD_PER_LUN, 111 .eh_timed_out = iscsi_eh_cmd_timed_out, 112 .eh_abort_handler = iscsi_eh_abort, 113 .eh_device_reset_handler = iscsi_eh_device_reset, 114 .eh_target_reset_handler = iscsi_eh_recover_target, 115 .target_alloc = iscsi_target_alloc, 116 .dma_boundary = PAGE_SIZE - 1, 117 .this_id = -1, 118 .track_queue_depth = 1, 119}; 120 121static struct iscsi_transport cxgb4i_iscsi_transport = { 122 .owner = THIS_MODULE, 123 .name = DRV_MODULE_NAME, 124 .caps = CAP_RECOVERY_L0 | CAP_MULTI_R2T | CAP_HDRDGST | 125 CAP_DATADGST | CAP_DIGEST_OFFLOAD | 126 CAP_PADDING_OFFLOAD | CAP_TEXT_NEGO, 127 .attr_is_visible = cxgbi_attr_is_visible, 128 .get_host_param = cxgbi_get_host_param, 129 .set_host_param = cxgbi_set_host_param, 130 /* session management */ 131 .create_session = cxgbi_create_session, 132 .destroy_session = cxgbi_destroy_session, 133 .get_session_param = iscsi_session_get_param, 134 /* connection management */ 135 .create_conn = cxgbi_create_conn, 136 .bind_conn = cxgbi_bind_conn, 137 .unbind_conn = iscsi_conn_unbind, 138 .destroy_conn = iscsi_tcp_conn_teardown, 139 .start_conn = iscsi_conn_start, 140 .stop_conn = iscsi_conn_stop, 141 .get_conn_param = iscsi_conn_get_param, 142 .set_param = cxgbi_set_conn_param, 143 .get_stats = cxgbi_get_conn_stats, 144 /* pdu xmit req from user space */ 145 .send_pdu = iscsi_conn_send_pdu, 146 /* task */ 147 .init_task = iscsi_tcp_task_init, 148 .xmit_task = iscsi_tcp_task_xmit, 149 .cleanup_task = cxgbi_cleanup_task, 150 /* pdu */ 151 .alloc_pdu = cxgbi_conn_alloc_pdu, 152 .init_pdu = cxgbi_conn_init_pdu, 153 .xmit_pdu = cxgbi_conn_xmit_pdu, 154 .parse_pdu_itt = cxgbi_parse_pdu_itt, 155 /* TCP connect/disconnect */ 156 .get_ep_param = cxgbi_get_ep_param, 157 .ep_connect = cxgbi_ep_connect, 158 .ep_poll = cxgbi_ep_poll, 159 .ep_disconnect = cxgbi_ep_disconnect, 160 /* Error recovery timeout call */ 161 .session_recovery_timedout = iscsi_session_recovery_timedout, 162}; 163 164#ifdef CONFIG_CHELSIO_T4_DCB 165static int 166cxgb4_dcb_change_notify(struct notifier_block *, unsigned long, void *); 167 168static struct notifier_block cxgb4_dcb_change = { 169 .notifier_call = cxgb4_dcb_change_notify, 170}; 171#endif 172 173static struct scsi_transport_template *cxgb4i_stt; 174 175/* 176 * CPL (Chelsio Protocol Language) defines a message passing interface between 177 * the host driver and Chelsio asic. 178 * The section below implments CPLs that related to iscsi tcp connection 179 * open/close/abort and data send/receive. 180 */ 181 182#define RCV_BUFSIZ_MASK 0x3FFU 183#define MAX_IMM_TX_PKT_LEN 256 184 185static int push_tx_frames(struct cxgbi_sock *, int); 186 187/* 188 * is_ofld_imm - check whether a packet can be sent as immediate data 189 * @skb: the packet 190 * 191 * Returns true if a packet can be sent as an offload WR with immediate 192 * data. We currently use the same limit as for Ethernet packets. 193 */ 194static inline bool is_ofld_imm(const struct sk_buff *skb) 195{ 196 int len = skb->len; 197 198 if (likely(cxgbi_skcb_test_flag(skb, SKCBF_TX_NEED_HDR))) 199 len += sizeof(struct fw_ofld_tx_data_wr); 200 201 if (likely(cxgbi_skcb_test_flag((struct sk_buff *)skb, SKCBF_TX_ISO))) 202 len += sizeof(struct cpl_tx_data_iso); 203 204 return (len <= MAX_IMM_OFLD_TX_DATA_WR_LEN); 205} 206 207static void send_act_open_req(struct cxgbi_sock *csk, struct sk_buff *skb, 208 struct l2t_entry *e) 209{ 210 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(csk->cdev); 211 int wscale = cxgbi_sock_compute_wscale(csk->mss_idx); 212 unsigned long long opt0; 213 unsigned int opt2; 214 unsigned int qid_atid = ((unsigned int)csk->atid) | 215 (((unsigned int)csk->rss_qid) << 14); 216 217 opt0 = KEEP_ALIVE_F | 218 WND_SCALE_V(wscale) | 219 MSS_IDX_V(csk->mss_idx) | 220 L2T_IDX_V(((struct l2t_entry *)csk->l2t)->idx) | 221 TX_CHAN_V(csk->tx_chan) | 222 SMAC_SEL_V(csk->smac_idx) | 223 ULP_MODE_V(ULP_MODE_ISCSI) | 224 RCV_BUFSIZ_V(csk->rcv_win >> 10); 225 226 opt2 = RX_CHANNEL_V(0) | 227 RSS_QUEUE_VALID_F | 228 RSS_QUEUE_V(csk->rss_qid); 229 230 if (is_t4(lldi->adapter_type)) { 231 struct cpl_act_open_req *req = 232 (struct cpl_act_open_req *)skb->head; 233 234 INIT_TP_WR(req, 0); 235 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, 236 qid_atid)); 237 req->local_port = csk->saddr.sin_port; 238 req->peer_port = csk->daddr.sin_port; 239 req->local_ip = csk->saddr.sin_addr.s_addr; 240 req->peer_ip = csk->daddr.sin_addr.s_addr; 241 req->opt0 = cpu_to_be64(opt0); 242 req->params = cpu_to_be32(cxgb4_select_ntuple( 243 csk->cdev->ports[csk->port_id], 244 csk->l2t)); 245 opt2 |= RX_FC_VALID_F; 246 req->opt2 = cpu_to_be32(opt2); 247 248 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 249 "csk t4 0x%p, %pI4:%u-%pI4:%u, atid %d, qid %u.\n", 250 csk, &req->local_ip, ntohs(req->local_port), 251 &req->peer_ip, ntohs(req->peer_port), 252 csk->atid, csk->rss_qid); 253 } else if (is_t5(lldi->adapter_type)) { 254 struct cpl_t5_act_open_req *req = 255 (struct cpl_t5_act_open_req *)skb->head; 256 u32 isn = (prandom_u32() & ~7UL) - 1; 257 258 INIT_TP_WR(req, 0); 259 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, 260 qid_atid)); 261 req->local_port = csk->saddr.sin_port; 262 req->peer_port = csk->daddr.sin_port; 263 req->local_ip = csk->saddr.sin_addr.s_addr; 264 req->peer_ip = csk->daddr.sin_addr.s_addr; 265 req->opt0 = cpu_to_be64(opt0); 266 req->params = cpu_to_be64(FILTER_TUPLE_V( 267 cxgb4_select_ntuple( 268 csk->cdev->ports[csk->port_id], 269 csk->l2t))); 270 req->rsvd = cpu_to_be32(isn); 271 opt2 |= T5_ISS_VALID; 272 opt2 |= T5_OPT_2_VALID_F; 273 274 req->opt2 = cpu_to_be32(opt2); 275 276 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 277 "csk t5 0x%p, %pI4:%u-%pI4:%u, atid %d, qid %u.\n", 278 csk, &req->local_ip, ntohs(req->local_port), 279 &req->peer_ip, ntohs(req->peer_port), 280 csk->atid, csk->rss_qid); 281 } else { 282 struct cpl_t6_act_open_req *req = 283 (struct cpl_t6_act_open_req *)skb->head; 284 u32 isn = (prandom_u32() & ~7UL) - 1; 285 286 INIT_TP_WR(req, 0); 287 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, 288 qid_atid)); 289 req->local_port = csk->saddr.sin_port; 290 req->peer_port = csk->daddr.sin_port; 291 req->local_ip = csk->saddr.sin_addr.s_addr; 292 req->peer_ip = csk->daddr.sin_addr.s_addr; 293 req->opt0 = cpu_to_be64(opt0); 294 req->params = cpu_to_be64(FILTER_TUPLE_V( 295 cxgb4_select_ntuple( 296 csk->cdev->ports[csk->port_id], 297 csk->l2t))); 298 req->rsvd = cpu_to_be32(isn); 299 300 opt2 |= T5_ISS_VALID; 301 opt2 |= RX_FC_DISABLE_F; 302 opt2 |= T5_OPT_2_VALID_F; 303 304 req->opt2 = cpu_to_be32(opt2); 305 req->rsvd2 = cpu_to_be32(0); 306 req->opt3 = cpu_to_be32(0); 307 308 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 309 "csk t6 0x%p, %pI4:%u-%pI4:%u, atid %d, qid %u.\n", 310 csk, &req->local_ip, ntohs(req->local_port), 311 &req->peer_ip, ntohs(req->peer_port), 312 csk->atid, csk->rss_qid); 313 } 314 315 set_wr_txq(skb, CPL_PRIORITY_SETUP, csk->port_id); 316 317 pr_info_ipaddr("t%d csk 0x%p,%u,0x%lx,%u, rss_qid %u.\n", 318 (&csk->saddr), (&csk->daddr), 319 CHELSIO_CHIP_VERSION(lldi->adapter_type), csk, 320 csk->state, csk->flags, csk->atid, csk->rss_qid); 321 322 cxgb4_l2t_send(csk->cdev->ports[csk->port_id], skb, csk->l2t); 323} 324 325#if IS_ENABLED(CONFIG_IPV6) 326static void send_act_open_req6(struct cxgbi_sock *csk, struct sk_buff *skb, 327 struct l2t_entry *e) 328{ 329 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(csk->cdev); 330 int wscale = cxgbi_sock_compute_wscale(csk->mss_idx); 331 unsigned long long opt0; 332 unsigned int opt2; 333 unsigned int qid_atid = ((unsigned int)csk->atid) | 334 (((unsigned int)csk->rss_qid) << 14); 335 336 opt0 = KEEP_ALIVE_F | 337 WND_SCALE_V(wscale) | 338 MSS_IDX_V(csk->mss_idx) | 339 L2T_IDX_V(((struct l2t_entry *)csk->l2t)->idx) | 340 TX_CHAN_V(csk->tx_chan) | 341 SMAC_SEL_V(csk->smac_idx) | 342 ULP_MODE_V(ULP_MODE_ISCSI) | 343 RCV_BUFSIZ_V(csk->rcv_win >> 10); 344 345 opt2 = RX_CHANNEL_V(0) | 346 RSS_QUEUE_VALID_F | 347 RSS_QUEUE_V(csk->rss_qid); 348 349 if (is_t4(lldi->adapter_type)) { 350 struct cpl_act_open_req6 *req = 351 (struct cpl_act_open_req6 *)skb->head; 352 353 INIT_TP_WR(req, 0); 354 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6, 355 qid_atid)); 356 req->local_port = csk->saddr6.sin6_port; 357 req->peer_port = csk->daddr6.sin6_port; 358 359 req->local_ip_hi = *(__be64 *)(csk->saddr6.sin6_addr.s6_addr); 360 req->local_ip_lo = *(__be64 *)(csk->saddr6.sin6_addr.s6_addr + 361 8); 362 req->peer_ip_hi = *(__be64 *)(csk->daddr6.sin6_addr.s6_addr); 363 req->peer_ip_lo = *(__be64 *)(csk->daddr6.sin6_addr.s6_addr + 364 8); 365 366 req->opt0 = cpu_to_be64(opt0); 367 368 opt2 |= RX_FC_VALID_F; 369 req->opt2 = cpu_to_be32(opt2); 370 371 req->params = cpu_to_be32(cxgb4_select_ntuple( 372 csk->cdev->ports[csk->port_id], 373 csk->l2t)); 374 } else if (is_t5(lldi->adapter_type)) { 375 struct cpl_t5_act_open_req6 *req = 376 (struct cpl_t5_act_open_req6 *)skb->head; 377 378 INIT_TP_WR(req, 0); 379 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6, 380 qid_atid)); 381 req->local_port = csk->saddr6.sin6_port; 382 req->peer_port = csk->daddr6.sin6_port; 383 req->local_ip_hi = *(__be64 *)(csk->saddr6.sin6_addr.s6_addr); 384 req->local_ip_lo = *(__be64 *)(csk->saddr6.sin6_addr.s6_addr + 385 8); 386 req->peer_ip_hi = *(__be64 *)(csk->daddr6.sin6_addr.s6_addr); 387 req->peer_ip_lo = *(__be64 *)(csk->daddr6.sin6_addr.s6_addr + 388 8); 389 req->opt0 = cpu_to_be64(opt0); 390 391 opt2 |= T5_OPT_2_VALID_F; 392 req->opt2 = cpu_to_be32(opt2); 393 394 req->params = cpu_to_be64(FILTER_TUPLE_V(cxgb4_select_ntuple( 395 csk->cdev->ports[csk->port_id], 396 csk->l2t))); 397 } else { 398 struct cpl_t6_act_open_req6 *req = 399 (struct cpl_t6_act_open_req6 *)skb->head; 400 401 INIT_TP_WR(req, 0); 402 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6, 403 qid_atid)); 404 req->local_port = csk->saddr6.sin6_port; 405 req->peer_port = csk->daddr6.sin6_port; 406 req->local_ip_hi = *(__be64 *)(csk->saddr6.sin6_addr.s6_addr); 407 req->local_ip_lo = *(__be64 *)(csk->saddr6.sin6_addr.s6_addr + 408 8); 409 req->peer_ip_hi = *(__be64 *)(csk->daddr6.sin6_addr.s6_addr); 410 req->peer_ip_lo = *(__be64 *)(csk->daddr6.sin6_addr.s6_addr + 411 8); 412 req->opt0 = cpu_to_be64(opt0); 413 414 opt2 |= RX_FC_DISABLE_F; 415 opt2 |= T5_OPT_2_VALID_F; 416 417 req->opt2 = cpu_to_be32(opt2); 418 419 req->params = cpu_to_be64(FILTER_TUPLE_V(cxgb4_select_ntuple( 420 csk->cdev->ports[csk->port_id], 421 csk->l2t))); 422 423 req->rsvd2 = cpu_to_be32(0); 424 req->opt3 = cpu_to_be32(0); 425 } 426 427 set_wr_txq(skb, CPL_PRIORITY_SETUP, csk->port_id); 428 429 pr_info("t%d csk 0x%p,%u,0x%lx,%u, [%pI6]:%u-[%pI6]:%u, rss_qid %u.\n", 430 CHELSIO_CHIP_VERSION(lldi->adapter_type), csk, csk->state, 431 csk->flags, csk->atid, 432 &csk->saddr6.sin6_addr, ntohs(csk->saddr.sin_port), 433 &csk->daddr6.sin6_addr, ntohs(csk->daddr.sin_port), 434 csk->rss_qid); 435 436 cxgb4_l2t_send(csk->cdev->ports[csk->port_id], skb, csk->l2t); 437} 438#endif 439 440static void send_close_req(struct cxgbi_sock *csk) 441{ 442 struct sk_buff *skb = csk->cpl_close; 443 struct cpl_close_con_req *req = (struct cpl_close_con_req *)skb->head; 444 unsigned int tid = csk->tid; 445 446 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 447 "csk 0x%p,%u,0x%lx, tid %u.\n", 448 csk, csk->state, csk->flags, csk->tid); 449 csk->cpl_close = NULL; 450 set_wr_txq(skb, CPL_PRIORITY_DATA, csk->port_id); 451 INIT_TP_WR(req, tid); 452 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid)); 453 req->rsvd = 0; 454 455 cxgbi_sock_skb_entail(csk, skb); 456 if (csk->state >= CTP_ESTABLISHED) 457 push_tx_frames(csk, 1); 458} 459 460static void abort_arp_failure(void *handle, struct sk_buff *skb) 461{ 462 struct cxgbi_sock *csk = (struct cxgbi_sock *)handle; 463 struct cpl_abort_req *req; 464 465 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 466 "csk 0x%p,%u,0x%lx, tid %u, abort.\n", 467 csk, csk->state, csk->flags, csk->tid); 468 req = (struct cpl_abort_req *)skb->data; 469 req->cmd = CPL_ABORT_NO_RST; 470 cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb); 471} 472 473static void send_abort_req(struct cxgbi_sock *csk) 474{ 475 struct cpl_abort_req *req; 476 struct sk_buff *skb = csk->cpl_abort_req; 477 478 if (unlikely(csk->state == CTP_ABORTING) || !skb || !csk->cdev) 479 return; 480 481 if (!cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT)) { 482 send_tx_flowc_wr(csk); 483 cxgbi_sock_set_flag(csk, CTPF_TX_DATA_SENT); 484 } 485 486 cxgbi_sock_set_state(csk, CTP_ABORTING); 487 cxgbi_sock_set_flag(csk, CTPF_ABORT_RPL_PENDING); 488 cxgbi_sock_purge_write_queue(csk); 489 490 csk->cpl_abort_req = NULL; 491 req = (struct cpl_abort_req *)skb->head; 492 set_wr_txq(skb, CPL_PRIORITY_DATA, csk->port_id); 493 req->cmd = CPL_ABORT_SEND_RST; 494 t4_set_arp_err_handler(skb, csk, abort_arp_failure); 495 INIT_TP_WR(req, csk->tid); 496 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_REQ, csk->tid)); 497 req->rsvd0 = htonl(csk->snd_nxt); 498 req->rsvd1 = !cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT); 499 500 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 501 "csk 0x%p,%u,0x%lx,%u, snd_nxt %u, 0x%x.\n", 502 csk, csk->state, csk->flags, csk->tid, csk->snd_nxt, 503 req->rsvd1); 504 505 cxgb4_l2t_send(csk->cdev->ports[csk->port_id], skb, csk->l2t); 506} 507 508static void send_abort_rpl(struct cxgbi_sock *csk, int rst_status) 509{ 510 struct sk_buff *skb = csk->cpl_abort_rpl; 511 struct cpl_abort_rpl *rpl = (struct cpl_abort_rpl *)skb->head; 512 513 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 514 "csk 0x%p,%u,0x%lx,%u, status %d.\n", 515 csk, csk->state, csk->flags, csk->tid, rst_status); 516 517 csk->cpl_abort_rpl = NULL; 518 set_wr_txq(skb, CPL_PRIORITY_DATA, csk->port_id); 519 INIT_TP_WR(rpl, csk->tid); 520 OPCODE_TID(rpl) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_RPL, csk->tid)); 521 rpl->cmd = rst_status; 522 cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb); 523} 524 525/* 526 * CPL connection rx data ack: host -> 527 * Send RX credits through an RX_DATA_ACK CPL message. Returns the number of 528 * credits sent. 529 */ 530static u32 send_rx_credits(struct cxgbi_sock *csk, u32 credits) 531{ 532 struct sk_buff *skb; 533 struct cpl_rx_data_ack *req; 534 535 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX, 536 "csk 0x%p,%u,0x%lx,%u, credit %u.\n", 537 csk, csk->state, csk->flags, csk->tid, credits); 538 539 skb = alloc_wr(sizeof(*req), 0, GFP_ATOMIC); 540 if (!skb) { 541 pr_info("csk 0x%p, credit %u, OOM.\n", csk, credits); 542 return 0; 543 } 544 req = (struct cpl_rx_data_ack *)skb->head; 545 546 set_wr_txq(skb, CPL_PRIORITY_ACK, csk->port_id); 547 INIT_TP_WR(req, csk->tid); 548 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_RX_DATA_ACK, 549 csk->tid)); 550 req->credit_dack = cpu_to_be32(RX_CREDITS_V(credits) 551 | RX_FORCE_ACK_F); 552 cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb); 553 return credits; 554} 555 556/* 557 * sgl_len - calculates the size of an SGL of the given capacity 558 * @n: the number of SGL entries 559 * Calculates the number of flits needed for a scatter/gather list that 560 * can hold the given number of entries. 561 */ 562static inline unsigned int sgl_len(unsigned int n) 563{ 564 n--; 565 return (3 * n) / 2 + (n & 1) + 2; 566} 567 568/* 569 * calc_tx_flits_ofld - calculate # of flits for an offload packet 570 * @skb: the packet 571 * 572 * Returns the number of flits needed for the given offload packet. 573 * These packets are already fully constructed and no additional headers 574 * will be added. 575 */ 576static inline unsigned int calc_tx_flits_ofld(const struct sk_buff *skb) 577{ 578 unsigned int flits, cnt; 579 580 if (is_ofld_imm(skb)) 581 return DIV_ROUND_UP(skb->len, 8); 582 flits = skb_transport_offset(skb) / 8; 583 cnt = skb_shinfo(skb)->nr_frags; 584 if (skb_tail_pointer(skb) != skb_transport_header(skb)) 585 cnt++; 586 return flits + sgl_len(cnt); 587} 588 589#define FLOWC_WR_NPARAMS_MIN 9 590static inline int tx_flowc_wr_credits(int *nparamsp, int *flowclenp) 591{ 592 int nparams, flowclen16, flowclen; 593 594 nparams = FLOWC_WR_NPARAMS_MIN; 595#ifdef CONFIG_CHELSIO_T4_DCB 596 nparams++; 597#endif 598 flowclen = offsetof(struct fw_flowc_wr, mnemval[nparams]); 599 flowclen16 = DIV_ROUND_UP(flowclen, 16); 600 flowclen = flowclen16 * 16; 601 /* 602 * Return the number of 16-byte credits used by the FlowC request. 603 * Pass back the nparams and actual FlowC length if requested. 604 */ 605 if (nparamsp) 606 *nparamsp = nparams; 607 if (flowclenp) 608 *flowclenp = flowclen; 609 610 return flowclen16; 611} 612 613static inline int send_tx_flowc_wr(struct cxgbi_sock *csk) 614{ 615 struct sk_buff *skb; 616 struct fw_flowc_wr *flowc; 617 int nparams, flowclen16, flowclen; 618 619#ifdef CONFIG_CHELSIO_T4_DCB 620 u16 vlan = ((struct l2t_entry *)csk->l2t)->vlan; 621#endif 622 flowclen16 = tx_flowc_wr_credits(&nparams, &flowclen); 623 skb = alloc_wr(flowclen, 0, GFP_ATOMIC); 624 flowc = (struct fw_flowc_wr *)skb->head; 625 flowc->op_to_nparams = 626 htonl(FW_WR_OP_V(FW_FLOWC_WR) | FW_FLOWC_WR_NPARAMS_V(nparams)); 627 flowc->flowid_len16 = 628 htonl(FW_WR_LEN16_V(flowclen16) | FW_WR_FLOWID_V(csk->tid)); 629 flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_PFNVFN; 630 flowc->mnemval[0].val = htonl(csk->cdev->pfvf); 631 flowc->mnemval[1].mnemonic = FW_FLOWC_MNEM_CH; 632 flowc->mnemval[1].val = htonl(csk->tx_chan); 633 flowc->mnemval[2].mnemonic = FW_FLOWC_MNEM_PORT; 634 flowc->mnemval[2].val = htonl(csk->tx_chan); 635 flowc->mnemval[3].mnemonic = FW_FLOWC_MNEM_IQID; 636 flowc->mnemval[3].val = htonl(csk->rss_qid); 637 flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_SNDNXT; 638 flowc->mnemval[4].val = htonl(csk->snd_nxt); 639 flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_RCVNXT; 640 flowc->mnemval[5].val = htonl(csk->rcv_nxt); 641 flowc->mnemval[6].mnemonic = FW_FLOWC_MNEM_SNDBUF; 642 flowc->mnemval[6].val = htonl(csk->snd_win); 643 flowc->mnemval[7].mnemonic = FW_FLOWC_MNEM_MSS; 644 flowc->mnemval[7].val = htonl(csk->advmss); 645 flowc->mnemval[8].mnemonic = 0; 646 flowc->mnemval[8].val = 0; 647 flowc->mnemval[8].mnemonic = FW_FLOWC_MNEM_TXDATAPLEN_MAX; 648 if (csk->cdev->skb_iso_txhdr) 649 flowc->mnemval[8].val = cpu_to_be32(CXGBI_MAX_ISO_DATA_IN_SKB); 650 else 651 flowc->mnemval[8].val = cpu_to_be32(16128); 652#ifdef CONFIG_CHELSIO_T4_DCB 653 flowc->mnemval[9].mnemonic = FW_FLOWC_MNEM_DCBPRIO; 654 if (vlan == CPL_L2T_VLAN_NONE) { 655 pr_warn_ratelimited("csk %u without VLAN Tag on DCB Link\n", 656 csk->tid); 657 flowc->mnemval[9].val = cpu_to_be32(0); 658 } else { 659 flowc->mnemval[9].val = cpu_to_be32((vlan & VLAN_PRIO_MASK) >> 660 VLAN_PRIO_SHIFT); 661 } 662#endif 663 664 set_wr_txq(skb, CPL_PRIORITY_DATA, csk->port_id); 665 666 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 667 "csk 0x%p, tid 0x%x, %u,%u,%u,%u,%u,%u,%u.\n", 668 csk, csk->tid, 0, csk->tx_chan, csk->rss_qid, 669 csk->snd_nxt, csk->rcv_nxt, csk->snd_win, 670 csk->advmss); 671 672 cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb); 673 674 return flowclen16; 675} 676 677static void 678cxgb4i_make_tx_iso_cpl(struct sk_buff *skb, struct cpl_tx_data_iso *cpl) 679{ 680 struct cxgbi_iso_info *info = (struct cxgbi_iso_info *)skb->head; 681 u32 imm_en = !!(info->flags & CXGBI_ISO_INFO_IMM_ENABLE); 682 u32 fslice = !!(info->flags & CXGBI_ISO_INFO_FSLICE); 683 u32 lslice = !!(info->flags & CXGBI_ISO_INFO_LSLICE); 684 u32 pdu_type = (info->op == ISCSI_OP_SCSI_CMD) ? 0 : 1; 685 u32 submode = cxgbi_skcb_tx_ulp_mode(skb) & 0x3; 686 687 cpl->op_to_scsi = cpu_to_be32(CPL_TX_DATA_ISO_OP_V(CPL_TX_DATA_ISO) | 688 CPL_TX_DATA_ISO_FIRST_V(fslice) | 689 CPL_TX_DATA_ISO_LAST_V(lslice) | 690 CPL_TX_DATA_ISO_CPLHDRLEN_V(0) | 691 CPL_TX_DATA_ISO_HDRCRC_V(submode & 1) | 692 CPL_TX_DATA_ISO_PLDCRC_V(((submode >> 1) & 1)) | 693 CPL_TX_DATA_ISO_IMMEDIATE_V(imm_en) | 694 CPL_TX_DATA_ISO_SCSI_V(pdu_type)); 695 696 cpl->ahs_len = info->ahs; 697 cpl->mpdu = cpu_to_be16(DIV_ROUND_UP(info->mpdu, 4)); 698 cpl->burst_size = cpu_to_be32(info->burst_size); 699 cpl->len = cpu_to_be32(info->len); 700 cpl->reserved2_seglen_offset = 701 cpu_to_be32(CPL_TX_DATA_ISO_SEGLEN_OFFSET_V(info->segment_offset)); 702 cpl->datasn_offset = cpu_to_be32(info->datasn_offset); 703 cpl->buffer_offset = cpu_to_be32(info->buffer_offset); 704 cpl->reserved3 = cpu_to_be32(0); 705 log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX, 706 "iso: flags 0x%x, op %u, ahs %u, num_pdu %u, mpdu %u, " 707 "burst_size %u, iso_len %u\n", 708 info->flags, info->op, info->ahs, info->num_pdu, 709 info->mpdu, info->burst_size << 2, info->len); 710} 711 712static void 713cxgb4i_make_tx_data_wr(struct cxgbi_sock *csk, struct sk_buff *skb, int dlen, 714 int len, u32 credits, int compl) 715{ 716 struct cxgbi_device *cdev = csk->cdev; 717 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); 718 struct fw_ofld_tx_data_wr *req; 719 struct cpl_tx_data_iso *cpl; 720 u32 submode = cxgbi_skcb_tx_ulp_mode(skb) & 0x3; 721 u32 wr_ulp_mode = 0; 722 u32 hdr_size = sizeof(*req); 723 u32 opcode = FW_OFLD_TX_DATA_WR; 724 u32 immlen = 0; 725 u32 force = is_t5(lldi->adapter_type) ? TX_FORCE_V(!submode) : 726 T6_TX_FORCE_F; 727 728 if (cxgbi_skcb_test_flag(skb, SKCBF_TX_ISO)) { 729 hdr_size += sizeof(struct cpl_tx_data_iso); 730 opcode = FW_ISCSI_TX_DATA_WR; 731 immlen += sizeof(struct cpl_tx_data_iso); 732 submode |= 8; 733 } 734 735 if (is_ofld_imm(skb)) 736 immlen += dlen; 737 738 req = (struct fw_ofld_tx_data_wr *)__skb_push(skb, hdr_size); 739 req->op_to_immdlen = cpu_to_be32(FW_WR_OP_V(opcode) | 740 FW_WR_COMPL_V(compl) | 741 FW_WR_IMMDLEN_V(immlen)); 742 req->flowid_len16 = cpu_to_be32(FW_WR_FLOWID_V(csk->tid) | 743 FW_WR_LEN16_V(credits)); 744 req->plen = cpu_to_be32(len); 745 cpl = (struct cpl_tx_data_iso *)(req + 1); 746 747 if (likely(cxgbi_skcb_test_flag(skb, SKCBF_TX_ISO))) 748 cxgb4i_make_tx_iso_cpl(skb, cpl); 749 750 if (submode) 751 wr_ulp_mode = FW_OFLD_TX_DATA_WR_ULPMODE_V(ULP2_MODE_ISCSI) | 752 FW_OFLD_TX_DATA_WR_ULPSUBMODE_V(submode); 753 754 req->tunnel_to_proxy = cpu_to_be32(wr_ulp_mode | force | 755 FW_OFLD_TX_DATA_WR_SHOVE_V(1U)); 756 757 if (!cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT)) 758 cxgbi_sock_set_flag(csk, CTPF_TX_DATA_SENT); 759} 760 761static void arp_failure_skb_discard(void *handle, struct sk_buff *skb) 762{ 763 kfree_skb(skb); 764} 765 766static int push_tx_frames(struct cxgbi_sock *csk, int req_completion) 767{ 768 int total_size = 0; 769 struct sk_buff *skb; 770 771 if (unlikely(csk->state < CTP_ESTABLISHED || 772 csk->state == CTP_CLOSE_WAIT_1 || csk->state >= CTP_ABORTING)) { 773 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK | 774 1 << CXGBI_DBG_PDU_TX, 775 "csk 0x%p,%u,0x%lx,%u, in closing state.\n", 776 csk, csk->state, csk->flags, csk->tid); 777 return 0; 778 } 779 780 while (csk->wr_cred && ((skb = skb_peek(&csk->write_queue)) != NULL)) { 781 struct cxgbi_iso_info *iso_cpl; 782 u32 dlen = skb->len; 783 u32 len = skb->len; 784 u32 iso_cpl_len = 0; 785 u32 flowclen16 = 0; 786 u32 credits_needed; 787 u32 num_pdu = 1, hdr_len; 788 789 if (cxgbi_skcb_test_flag(skb, SKCBF_TX_ISO)) 790 iso_cpl_len = sizeof(struct cpl_tx_data_iso); 791 792 if (is_ofld_imm(skb)) 793 credits_needed = DIV_ROUND_UP(dlen + iso_cpl_len, 16); 794 else 795 credits_needed = 796 DIV_ROUND_UP((8 * calc_tx_flits_ofld(skb)) + 797 iso_cpl_len, 16); 798 799 if (likely(cxgbi_skcb_test_flag(skb, SKCBF_TX_NEED_HDR))) 800 credits_needed += 801 DIV_ROUND_UP(sizeof(struct fw_ofld_tx_data_wr), 16); 802 803 /* 804 * Assumes the initial credits is large enough to support 805 * fw_flowc_wr plus largest possible first payload 806 */ 807 if (!cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT)) { 808 flowclen16 = send_tx_flowc_wr(csk); 809 csk->wr_cred -= flowclen16; 810 csk->wr_una_cred += flowclen16; 811 cxgbi_sock_set_flag(csk, CTPF_TX_DATA_SENT); 812 } 813 814 if (csk->wr_cred < credits_needed) { 815 log_debug(1 << CXGBI_DBG_PDU_TX, 816 "csk 0x%p, skb %u/%u, wr %d < %u.\n", 817 csk, skb->len, skb->data_len, 818 credits_needed, csk->wr_cred); 819 820 csk->no_tx_credits++; 821 break; 822 } 823 824 csk->no_tx_credits = 0; 825 826 __skb_unlink(skb, &csk->write_queue); 827 set_wr_txq(skb, CPL_PRIORITY_DATA, csk->port_id); 828 skb->csum = (__force __wsum)(credits_needed + flowclen16); 829 csk->wr_cred -= credits_needed; 830 csk->wr_una_cred += credits_needed; 831 cxgbi_sock_enqueue_wr(csk, skb); 832 833 log_debug(1 << CXGBI_DBG_PDU_TX, 834 "csk 0x%p, skb %u/%u, wr %d, left %u, unack %u.\n", 835 csk, skb->len, skb->data_len, credits_needed, 836 csk->wr_cred, csk->wr_una_cred); 837 838 if (!req_completion && 839 ((csk->wr_una_cred >= (csk->wr_max_cred / 2)) || 840 after(csk->write_seq, (csk->snd_una + csk->snd_win / 2)))) 841 req_completion = 1; 842 843 if (likely(cxgbi_skcb_test_flag(skb, SKCBF_TX_NEED_HDR))) { 844 u32 ulp_mode = cxgbi_skcb_tx_ulp_mode(skb); 845 846 if (cxgbi_skcb_test_flag(skb, SKCBF_TX_ISO)) { 847 iso_cpl = (struct cxgbi_iso_info *)skb->head; 848 num_pdu = iso_cpl->num_pdu; 849 hdr_len = cxgbi_skcb_tx_iscsi_hdrlen(skb); 850 len += (cxgbi_ulp_extra_len(ulp_mode) * num_pdu) + 851 (hdr_len * (num_pdu - 1)); 852 } else { 853 len += cxgbi_ulp_extra_len(ulp_mode); 854 } 855 856 cxgb4i_make_tx_data_wr(csk, skb, dlen, len, 857 credits_needed, req_completion); 858 csk->snd_nxt += len; 859 cxgbi_skcb_clear_flag(skb, SKCBF_TX_NEED_HDR); 860 } else if (cxgbi_skcb_test_flag(skb, SKCBF_TX_FLAG_COMPL) && 861 (csk->wr_una_cred >= (csk->wr_max_cred / 2))) { 862 struct cpl_close_con_req *req = 863 (struct cpl_close_con_req *)skb->data; 864 865 req->wr.wr_hi |= cpu_to_be32(FW_WR_COMPL_F); 866 } 867 868 total_size += skb->truesize; 869 t4_set_arp_err_handler(skb, csk, arp_failure_skb_discard); 870 871 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_TX, 872 "csk 0x%p,%u,0x%lx,%u, skb 0x%p, %u.\n", 873 csk, csk->state, csk->flags, csk->tid, skb, len); 874 cxgb4_l2t_send(csk->cdev->ports[csk->port_id], skb, csk->l2t); 875 } 876 return total_size; 877} 878 879static inline void free_atid(struct cxgbi_sock *csk) 880{ 881 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(csk->cdev); 882 883 if (cxgbi_sock_flag(csk, CTPF_HAS_ATID)) { 884 cxgb4_free_atid(lldi->tids, csk->atid); 885 cxgbi_sock_clear_flag(csk, CTPF_HAS_ATID); 886 cxgbi_sock_put(csk); 887 } 888} 889 890static void do_act_establish(struct cxgbi_device *cdev, struct sk_buff *skb) 891{ 892 struct cxgbi_sock *csk; 893 struct cpl_act_establish *req = (struct cpl_act_establish *)skb->data; 894 unsigned short tcp_opt = ntohs(req->tcp_opt); 895 unsigned int tid = GET_TID(req); 896 unsigned int atid = TID_TID_G(ntohl(req->tos_atid)); 897 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); 898 struct tid_info *t = lldi->tids; 899 u32 rcv_isn = be32_to_cpu(req->rcv_isn); 900 901 csk = lookup_atid(t, atid); 902 if (unlikely(!csk)) { 903 pr_err("NO conn. for atid %u, cdev 0x%p.\n", atid, cdev); 904 goto rel_skb; 905 } 906 907 if (csk->atid != atid) { 908 pr_err("bad conn atid %u, csk 0x%p,%u,0x%lx,tid %u, atid %u.\n", 909 atid, csk, csk->state, csk->flags, csk->tid, csk->atid); 910 goto rel_skb; 911 } 912 913 pr_info_ipaddr("atid 0x%x, tid 0x%x, csk 0x%p,%u,0x%lx, isn %u.\n", 914 (&csk->saddr), (&csk->daddr), 915 atid, tid, csk, csk->state, csk->flags, rcv_isn); 916 917 module_put(cdev->owner); 918 919 cxgbi_sock_get(csk); 920 csk->tid = tid; 921 cxgb4_insert_tid(lldi->tids, csk, tid, csk->csk_family); 922 cxgbi_sock_set_flag(csk, CTPF_HAS_TID); 923 924 free_atid(csk); 925 926 spin_lock_bh(&csk->lock); 927 if (unlikely(csk->state != CTP_ACTIVE_OPEN)) 928 pr_info("csk 0x%p,%u,0x%lx,%u, got EST.\n", 929 csk, csk->state, csk->flags, csk->tid); 930 931 if (csk->retry_timer.function) { 932 del_timer(&csk->retry_timer); 933 csk->retry_timer.function = NULL; 934 } 935 936 csk->copied_seq = csk->rcv_wup = csk->rcv_nxt = rcv_isn; 937 /* 938 * Causes the first RX_DATA_ACK to supply any Rx credits we couldn't 939 * pass through opt0. 940 */ 941 if (csk->rcv_win > (RCV_BUFSIZ_MASK << 10)) 942 csk->rcv_wup -= csk->rcv_win - (RCV_BUFSIZ_MASK << 10); 943 944 csk->advmss = lldi->mtus[TCPOPT_MSS_G(tcp_opt)] - 40; 945 if (TCPOPT_TSTAMP_G(tcp_opt)) 946 csk->advmss -= 12; 947 if (csk->advmss < 128) 948 csk->advmss = 128; 949 950 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 951 "csk 0x%p, mss_idx %u, advmss %u.\n", 952 csk, TCPOPT_MSS_G(tcp_opt), csk->advmss); 953 954 cxgbi_sock_established(csk, ntohl(req->snd_isn), ntohs(req->tcp_opt)); 955 956 if (unlikely(cxgbi_sock_flag(csk, CTPF_ACTIVE_CLOSE_NEEDED))) 957 send_abort_req(csk); 958 else { 959 if (skb_queue_len(&csk->write_queue)) 960 push_tx_frames(csk, 0); 961 cxgbi_conn_tx_open(csk); 962 } 963 spin_unlock_bh(&csk->lock); 964 965rel_skb: 966 __kfree_skb(skb); 967} 968 969static int act_open_rpl_status_to_errno(int status) 970{ 971 switch (status) { 972 case CPL_ERR_CONN_RESET: 973 return -ECONNREFUSED; 974 case CPL_ERR_ARP_MISS: 975 return -EHOSTUNREACH; 976 case CPL_ERR_CONN_TIMEDOUT: 977 return -ETIMEDOUT; 978 case CPL_ERR_TCAM_FULL: 979 return -ENOMEM; 980 case CPL_ERR_CONN_EXIST: 981 return -EADDRINUSE; 982 default: 983 return -EIO; 984 } 985} 986 987static void csk_act_open_retry_timer(struct timer_list *t) 988{ 989 struct sk_buff *skb = NULL; 990 struct cxgbi_sock *csk = from_timer(csk, t, retry_timer); 991 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(csk->cdev); 992 void (*send_act_open_func)(struct cxgbi_sock *, struct sk_buff *, 993 struct l2t_entry *); 994 int t4 = is_t4(lldi->adapter_type), size, size6; 995 996 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 997 "csk 0x%p,%u,0x%lx,%u.\n", 998 csk, csk->state, csk->flags, csk->tid); 999 1000 cxgbi_sock_get(csk); 1001 spin_lock_bh(&csk->lock); 1002 1003 if (t4) { 1004 size = sizeof(struct cpl_act_open_req); 1005 size6 = sizeof(struct cpl_act_open_req6); 1006 } else { 1007 size = sizeof(struct cpl_t5_act_open_req); 1008 size6 = sizeof(struct cpl_t5_act_open_req6); 1009 } 1010 1011 if (csk->csk_family == AF_INET) { 1012 send_act_open_func = send_act_open_req; 1013 skb = alloc_wr(size, 0, GFP_ATOMIC); 1014#if IS_ENABLED(CONFIG_IPV6) 1015 } else { 1016 send_act_open_func = send_act_open_req6; 1017 skb = alloc_wr(size6, 0, GFP_ATOMIC); 1018#endif 1019 } 1020 1021 if (!skb) 1022 cxgbi_sock_fail_act_open(csk, -ENOMEM); 1023 else { 1024 skb->sk = (struct sock *)csk; 1025 t4_set_arp_err_handler(skb, csk, 1026 cxgbi_sock_act_open_req_arp_failure); 1027 send_act_open_func(csk, skb, csk->l2t); 1028 } 1029 1030 spin_unlock_bh(&csk->lock); 1031 cxgbi_sock_put(csk); 1032 1033} 1034 1035static inline bool is_neg_adv(unsigned int status) 1036{ 1037 return status == CPL_ERR_RTX_NEG_ADVICE || 1038 status == CPL_ERR_KEEPALV_NEG_ADVICE || 1039 status == CPL_ERR_PERSIST_NEG_ADVICE; 1040} 1041 1042static void do_act_open_rpl(struct cxgbi_device *cdev, struct sk_buff *skb) 1043{ 1044 struct cxgbi_sock *csk; 1045 struct cpl_act_open_rpl *rpl = (struct cpl_act_open_rpl *)skb->data; 1046 unsigned int tid = GET_TID(rpl); 1047 unsigned int atid = 1048 TID_TID_G(AOPEN_ATID_G(be32_to_cpu(rpl->atid_status))); 1049 unsigned int status = AOPEN_STATUS_G(be32_to_cpu(rpl->atid_status)); 1050 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); 1051 struct tid_info *t = lldi->tids; 1052 1053 csk = lookup_atid(t, atid); 1054 if (unlikely(!csk)) { 1055 pr_err("NO matching conn. atid %u, tid %u.\n", atid, tid); 1056 goto rel_skb; 1057 } 1058 1059 pr_info_ipaddr("tid %u/%u, status %u.\n" 1060 "csk 0x%p,%u,0x%lx. ", (&csk->saddr), (&csk->daddr), 1061 atid, tid, status, csk, csk->state, csk->flags); 1062 1063 if (is_neg_adv(status)) 1064 goto rel_skb; 1065 1066 module_put(cdev->owner); 1067 1068 if (status && status != CPL_ERR_TCAM_FULL && 1069 status != CPL_ERR_CONN_EXIST && 1070 status != CPL_ERR_ARP_MISS) 1071 cxgb4_remove_tid(lldi->tids, csk->port_id, GET_TID(rpl), 1072 csk->csk_family); 1073 1074 cxgbi_sock_get(csk); 1075 spin_lock_bh(&csk->lock); 1076 1077 if (status == CPL_ERR_CONN_EXIST && 1078 csk->retry_timer.function != csk_act_open_retry_timer) { 1079 csk->retry_timer.function = csk_act_open_retry_timer; 1080 mod_timer(&csk->retry_timer, jiffies + HZ / 2); 1081 } else 1082 cxgbi_sock_fail_act_open(csk, 1083 act_open_rpl_status_to_errno(status)); 1084 1085 spin_unlock_bh(&csk->lock); 1086 cxgbi_sock_put(csk); 1087rel_skb: 1088 __kfree_skb(skb); 1089} 1090 1091static void do_peer_close(struct cxgbi_device *cdev, struct sk_buff *skb) 1092{ 1093 struct cxgbi_sock *csk; 1094 struct cpl_peer_close *req = (struct cpl_peer_close *)skb->data; 1095 unsigned int tid = GET_TID(req); 1096 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); 1097 struct tid_info *t = lldi->tids; 1098 1099 csk = lookup_tid(t, tid); 1100 if (unlikely(!csk)) { 1101 pr_err("can't find connection for tid %u.\n", tid); 1102 goto rel_skb; 1103 } 1104 pr_info_ipaddr("csk 0x%p,%u,0x%lx,%u.\n", 1105 (&csk->saddr), (&csk->daddr), 1106 csk, csk->state, csk->flags, csk->tid); 1107 cxgbi_sock_rcv_peer_close(csk); 1108rel_skb: 1109 __kfree_skb(skb); 1110} 1111 1112static void do_close_con_rpl(struct cxgbi_device *cdev, struct sk_buff *skb) 1113{ 1114 struct cxgbi_sock *csk; 1115 struct cpl_close_con_rpl *rpl = (struct cpl_close_con_rpl *)skb->data; 1116 unsigned int tid = GET_TID(rpl); 1117 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); 1118 struct tid_info *t = lldi->tids; 1119 1120 csk = lookup_tid(t, tid); 1121 if (unlikely(!csk)) { 1122 pr_err("can't find connection for tid %u.\n", tid); 1123 goto rel_skb; 1124 } 1125 pr_info_ipaddr("csk 0x%p,%u,0x%lx,%u.\n", 1126 (&csk->saddr), (&csk->daddr), 1127 csk, csk->state, csk->flags, csk->tid); 1128 cxgbi_sock_rcv_close_conn_rpl(csk, ntohl(rpl->snd_nxt)); 1129rel_skb: 1130 __kfree_skb(skb); 1131} 1132 1133static int abort_status_to_errno(struct cxgbi_sock *csk, int abort_reason, 1134 int *need_rst) 1135{ 1136 switch (abort_reason) { 1137 case CPL_ERR_BAD_SYN: 1138 case CPL_ERR_CONN_RESET: 1139 return csk->state > CTP_ESTABLISHED ? 1140 -EPIPE : -ECONNRESET; 1141 case CPL_ERR_XMIT_TIMEDOUT: 1142 case CPL_ERR_PERSIST_TIMEDOUT: 1143 case CPL_ERR_FINWAIT2_TIMEDOUT: 1144 case CPL_ERR_KEEPALIVE_TIMEDOUT: 1145 return -ETIMEDOUT; 1146 default: 1147 return -EIO; 1148 } 1149} 1150 1151static void do_abort_req_rss(struct cxgbi_device *cdev, struct sk_buff *skb) 1152{ 1153 struct cxgbi_sock *csk; 1154 struct cpl_abort_req_rss *req = (struct cpl_abort_req_rss *)skb->data; 1155 unsigned int tid = GET_TID(req); 1156 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); 1157 struct tid_info *t = lldi->tids; 1158 int rst_status = CPL_ABORT_NO_RST; 1159 1160 csk = lookup_tid(t, tid); 1161 if (unlikely(!csk)) { 1162 pr_err("can't find connection for tid %u.\n", tid); 1163 goto rel_skb; 1164 } 1165 1166 pr_info_ipaddr("csk 0x%p,%u,0x%lx,%u, status %u.\n", 1167 (&csk->saddr), (&csk->daddr), 1168 csk, csk->state, csk->flags, csk->tid, req->status); 1169 1170 if (is_neg_adv(req->status)) 1171 goto rel_skb; 1172 1173 cxgbi_sock_get(csk); 1174 spin_lock_bh(&csk->lock); 1175 1176 cxgbi_sock_clear_flag(csk, CTPF_ABORT_REQ_RCVD); 1177 1178 if (!cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT)) { 1179 send_tx_flowc_wr(csk); 1180 cxgbi_sock_set_flag(csk, CTPF_TX_DATA_SENT); 1181 } 1182 1183 cxgbi_sock_set_flag(csk, CTPF_ABORT_REQ_RCVD); 1184 cxgbi_sock_set_state(csk, CTP_ABORTING); 1185 1186 send_abort_rpl(csk, rst_status); 1187 1188 if (!cxgbi_sock_flag(csk, CTPF_ABORT_RPL_PENDING)) { 1189 csk->err = abort_status_to_errno(csk, req->status, &rst_status); 1190 cxgbi_sock_closed(csk); 1191 } 1192 1193 spin_unlock_bh(&csk->lock); 1194 cxgbi_sock_put(csk); 1195rel_skb: 1196 __kfree_skb(skb); 1197} 1198 1199static void do_abort_rpl_rss(struct cxgbi_device *cdev, struct sk_buff *skb) 1200{ 1201 struct cxgbi_sock *csk; 1202 struct cpl_abort_rpl_rss *rpl = (struct cpl_abort_rpl_rss *)skb->data; 1203 unsigned int tid = GET_TID(rpl); 1204 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); 1205 struct tid_info *t = lldi->tids; 1206 1207 csk = lookup_tid(t, tid); 1208 if (!csk) 1209 goto rel_skb; 1210 1211 pr_info_ipaddr("csk 0x%p,%u,0x%lx,%u, status %u.\n", 1212 (&csk->saddr), (&csk->daddr), csk, 1213 csk->state, csk->flags, csk->tid, rpl->status); 1214 1215 if (rpl->status == CPL_ERR_ABORT_FAILED) 1216 goto rel_skb; 1217 1218 cxgbi_sock_rcv_abort_rpl(csk); 1219rel_skb: 1220 __kfree_skb(skb); 1221} 1222 1223static void do_rx_data(struct cxgbi_device *cdev, struct sk_buff *skb) 1224{ 1225 struct cxgbi_sock *csk; 1226 struct cpl_rx_data *cpl = (struct cpl_rx_data *)skb->data; 1227 unsigned int tid = GET_TID(cpl); 1228 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); 1229 struct tid_info *t = lldi->tids; 1230 1231 csk = lookup_tid(t, tid); 1232 if (!csk) { 1233 pr_err("can't find connection for tid %u.\n", tid); 1234 } else { 1235 /* not expecting this, reset the connection. */ 1236 pr_err("csk 0x%p, tid %u, rcv cpl_rx_data.\n", csk, tid); 1237 spin_lock_bh(&csk->lock); 1238 send_abort_req(csk); 1239 spin_unlock_bh(&csk->lock); 1240 } 1241 __kfree_skb(skb); 1242} 1243 1244static void do_rx_iscsi_hdr(struct cxgbi_device *cdev, struct sk_buff *skb) 1245{ 1246 struct cxgbi_sock *csk; 1247 struct cpl_iscsi_hdr *cpl = (struct cpl_iscsi_hdr *)skb->data; 1248 unsigned short pdu_len_ddp = be16_to_cpu(cpl->pdu_len_ddp); 1249 unsigned int tid = GET_TID(cpl); 1250 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); 1251 struct tid_info *t = lldi->tids; 1252 1253 csk = lookup_tid(t, tid); 1254 if (unlikely(!csk)) { 1255 pr_err("can't find conn. for tid %u.\n", tid); 1256 goto rel_skb; 1257 } 1258 1259 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX, 1260 "csk 0x%p,%u,0x%lx, tid %u, skb 0x%p,%u, 0x%x.\n", 1261 csk, csk->state, csk->flags, csk->tid, skb, skb->len, 1262 pdu_len_ddp); 1263 1264 spin_lock_bh(&csk->lock); 1265 1266 if (unlikely(csk->state >= CTP_PASSIVE_CLOSE)) { 1267 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 1268 "csk 0x%p,%u,0x%lx,%u, bad state.\n", 1269 csk, csk->state, csk->flags, csk->tid); 1270 if (csk->state != CTP_ABORTING) 1271 goto abort_conn; 1272 else 1273 goto discard; 1274 } 1275 1276 cxgbi_skcb_tcp_seq(skb) = ntohl(cpl->seq); 1277 cxgbi_skcb_flags(skb) = 0; 1278 1279 skb_reset_transport_header(skb); 1280 __skb_pull(skb, sizeof(*cpl)); 1281 __pskb_trim(skb, ntohs(cpl->len)); 1282 1283 if (!csk->skb_ulp_lhdr) { 1284 unsigned char *bhs; 1285 unsigned int hlen, dlen, plen; 1286 1287 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX, 1288 "csk 0x%p,%u,0x%lx, tid %u, skb 0x%p header.\n", 1289 csk, csk->state, csk->flags, csk->tid, skb); 1290 csk->skb_ulp_lhdr = skb; 1291 cxgbi_skcb_set_flag(skb, SKCBF_RX_HDR); 1292 1293 if ((CHELSIO_CHIP_VERSION(lldi->adapter_type) <= CHELSIO_T5) && 1294 (cxgbi_skcb_tcp_seq(skb) != csk->rcv_nxt)) { 1295 pr_info("tid %u, CPL_ISCSI_HDR, bad seq, 0x%x/0x%x.\n", 1296 csk->tid, cxgbi_skcb_tcp_seq(skb), 1297 csk->rcv_nxt); 1298 goto abort_conn; 1299 } 1300 1301 bhs = skb->data; 1302 hlen = ntohs(cpl->len); 1303 dlen = ntohl(*(unsigned int *)(bhs + 4)) & 0xFFFFFF; 1304 1305 plen = ISCSI_PDU_LEN_G(pdu_len_ddp); 1306 if (is_t4(lldi->adapter_type)) 1307 plen -= 40; 1308 1309 if ((hlen + dlen) != plen) { 1310 pr_info("tid 0x%x, CPL_ISCSI_HDR, pdu len " 1311 "mismatch %u != %u + %u, seq 0x%x.\n", 1312 csk->tid, plen, hlen, dlen, 1313 cxgbi_skcb_tcp_seq(skb)); 1314 goto abort_conn; 1315 } 1316 1317 cxgbi_skcb_rx_pdulen(skb) = (hlen + dlen + 3) & (~0x3); 1318 if (dlen) 1319 cxgbi_skcb_rx_pdulen(skb) += csk->dcrc_len; 1320 csk->rcv_nxt += cxgbi_skcb_rx_pdulen(skb); 1321 1322 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX, 1323 "csk 0x%p, skb 0x%p, 0x%x,%u+%u,0x%x,0x%x.\n", 1324 csk, skb, *bhs, hlen, dlen, 1325 ntohl(*((unsigned int *)(bhs + 16))), 1326 ntohl(*((unsigned int *)(bhs + 24)))); 1327 1328 } else { 1329 struct sk_buff *lskb = csk->skb_ulp_lhdr; 1330 1331 cxgbi_skcb_set_flag(lskb, SKCBF_RX_DATA); 1332 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX, 1333 "csk 0x%p,%u,0x%lx, skb 0x%p data, 0x%p.\n", 1334 csk, csk->state, csk->flags, skb, lskb); 1335 } 1336 1337 __skb_queue_tail(&csk->receive_queue, skb); 1338 spin_unlock_bh(&csk->lock); 1339 return; 1340 1341abort_conn: 1342 send_abort_req(csk); 1343discard: 1344 spin_unlock_bh(&csk->lock); 1345rel_skb: 1346 __kfree_skb(skb); 1347} 1348 1349static void do_rx_iscsi_data(struct cxgbi_device *cdev, struct sk_buff *skb) 1350{ 1351 struct cxgbi_sock *csk; 1352 struct cpl_iscsi_hdr *cpl = (struct cpl_iscsi_hdr *)skb->data; 1353 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); 1354 struct tid_info *t = lldi->tids; 1355 struct sk_buff *lskb; 1356 u32 tid = GET_TID(cpl); 1357 u16 pdu_len_ddp = be16_to_cpu(cpl->pdu_len_ddp); 1358 1359 csk = lookup_tid(t, tid); 1360 if (unlikely(!csk)) { 1361 pr_err("can't find conn. for tid %u.\n", tid); 1362 goto rel_skb; 1363 } 1364 1365 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX, 1366 "csk 0x%p,%u,0x%lx, tid %u, skb 0x%p,%u, 0x%x.\n", 1367 csk, csk->state, csk->flags, csk->tid, skb, 1368 skb->len, pdu_len_ddp); 1369 1370 spin_lock_bh(&csk->lock); 1371 1372 if (unlikely(csk->state >= CTP_PASSIVE_CLOSE)) { 1373 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 1374 "csk 0x%p,%u,0x%lx,%u, bad state.\n", 1375 csk, csk->state, csk->flags, csk->tid); 1376 1377 if (csk->state != CTP_ABORTING) 1378 goto abort_conn; 1379 else 1380 goto discard; 1381 } 1382 1383 cxgbi_skcb_tcp_seq(skb) = be32_to_cpu(cpl->seq); 1384 cxgbi_skcb_flags(skb) = 0; 1385 1386 skb_reset_transport_header(skb); 1387 __skb_pull(skb, sizeof(*cpl)); 1388 __pskb_trim(skb, ntohs(cpl->len)); 1389 1390 if (!csk->skb_ulp_lhdr) 1391 csk->skb_ulp_lhdr = skb; 1392 1393 lskb = csk->skb_ulp_lhdr; 1394 cxgbi_skcb_set_flag(lskb, SKCBF_RX_DATA); 1395 1396 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX, 1397 "csk 0x%p,%u,0x%lx, skb 0x%p data, 0x%p.\n", 1398 csk, csk->state, csk->flags, skb, lskb); 1399 1400 __skb_queue_tail(&csk->receive_queue, skb); 1401 spin_unlock_bh(&csk->lock); 1402 return; 1403 1404abort_conn: 1405 send_abort_req(csk); 1406discard: 1407 spin_unlock_bh(&csk->lock); 1408rel_skb: 1409 __kfree_skb(skb); 1410} 1411 1412static void 1413cxgb4i_process_ddpvld(struct cxgbi_sock *csk, 1414 struct sk_buff *skb, u32 ddpvld) 1415{ 1416 if (ddpvld & (1 << CPL_RX_DDP_STATUS_HCRC_SHIFT)) { 1417 pr_info("csk 0x%p, lhdr 0x%p, status 0x%x, hcrc bad 0x%lx.\n", 1418 csk, skb, ddpvld, cxgbi_skcb_flags(skb)); 1419 cxgbi_skcb_set_flag(skb, SKCBF_RX_HCRC_ERR); 1420 } 1421 1422 if (ddpvld & (1 << CPL_RX_DDP_STATUS_DCRC_SHIFT)) { 1423 pr_info("csk 0x%p, lhdr 0x%p, status 0x%x, dcrc bad 0x%lx.\n", 1424 csk, skb, ddpvld, cxgbi_skcb_flags(skb)); 1425 cxgbi_skcb_set_flag(skb, SKCBF_RX_DCRC_ERR); 1426 } 1427 1428 if (ddpvld & (1 << CPL_RX_DDP_STATUS_PAD_SHIFT)) { 1429 log_debug(1 << CXGBI_DBG_PDU_RX, 1430 "csk 0x%p, lhdr 0x%p, status 0x%x, pad bad.\n", 1431 csk, skb, ddpvld); 1432 cxgbi_skcb_set_flag(skb, SKCBF_RX_PAD_ERR); 1433 } 1434 1435 if ((ddpvld & (1 << CPL_RX_DDP_STATUS_DDP_SHIFT)) && 1436 !cxgbi_skcb_test_flag(skb, SKCBF_RX_DATA)) { 1437 log_debug(1 << CXGBI_DBG_PDU_RX, 1438 "csk 0x%p, lhdr 0x%p, 0x%x, data ddp'ed.\n", 1439 csk, skb, ddpvld); 1440 cxgbi_skcb_set_flag(skb, SKCBF_RX_DATA_DDPD); 1441 } 1442} 1443 1444static void do_rx_data_ddp(struct cxgbi_device *cdev, 1445 struct sk_buff *skb) 1446{ 1447 struct cxgbi_sock *csk; 1448 struct sk_buff *lskb; 1449 struct cpl_rx_data_ddp *rpl = (struct cpl_rx_data_ddp *)skb->data; 1450 unsigned int tid = GET_TID(rpl); 1451 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); 1452 struct tid_info *t = lldi->tids; 1453 u32 ddpvld = be32_to_cpu(rpl->ddpvld); 1454 1455 csk = lookup_tid(t, tid); 1456 if (unlikely(!csk)) { 1457 pr_err("can't find connection for tid %u.\n", tid); 1458 goto rel_skb; 1459 } 1460 1461 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX, 1462 "csk 0x%p,%u,0x%lx, skb 0x%p,0x%x, lhdr 0x%p.\n", 1463 csk, csk->state, csk->flags, skb, ddpvld, csk->skb_ulp_lhdr); 1464 1465 spin_lock_bh(&csk->lock); 1466 1467 if (unlikely(csk->state >= CTP_PASSIVE_CLOSE)) { 1468 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 1469 "csk 0x%p,%u,0x%lx,%u, bad state.\n", 1470 csk, csk->state, csk->flags, csk->tid); 1471 if (csk->state != CTP_ABORTING) 1472 goto abort_conn; 1473 else 1474 goto discard; 1475 } 1476 1477 if (!csk->skb_ulp_lhdr) { 1478 pr_err("tid 0x%x, rcv RX_DATA_DDP w/o pdu bhs.\n", csk->tid); 1479 goto abort_conn; 1480 } 1481 1482 lskb = csk->skb_ulp_lhdr; 1483 csk->skb_ulp_lhdr = NULL; 1484 1485 cxgbi_skcb_rx_ddigest(lskb) = ntohl(rpl->ulp_crc); 1486 1487 if (ntohs(rpl->len) != cxgbi_skcb_rx_pdulen(lskb)) 1488 pr_info("tid 0x%x, RX_DATA_DDP pdulen %u != %u.\n", 1489 csk->tid, ntohs(rpl->len), cxgbi_skcb_rx_pdulen(lskb)); 1490 1491 cxgb4i_process_ddpvld(csk, lskb, ddpvld); 1492 1493 log_debug(1 << CXGBI_DBG_PDU_RX, 1494 "csk 0x%p, lskb 0x%p, f 0x%lx.\n", 1495 csk, lskb, cxgbi_skcb_flags(lskb)); 1496 1497 cxgbi_skcb_set_flag(lskb, SKCBF_RX_STATUS); 1498 cxgbi_conn_pdu_ready(csk); 1499 spin_unlock_bh(&csk->lock); 1500 goto rel_skb; 1501 1502abort_conn: 1503 send_abort_req(csk); 1504discard: 1505 spin_unlock_bh(&csk->lock); 1506rel_skb: 1507 __kfree_skb(skb); 1508} 1509 1510static void 1511do_rx_iscsi_cmp(struct cxgbi_device *cdev, struct sk_buff *skb) 1512{ 1513 struct cxgbi_sock *csk; 1514 struct cpl_rx_iscsi_cmp *rpl = (struct cpl_rx_iscsi_cmp *)skb->data; 1515 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); 1516 struct tid_info *t = lldi->tids; 1517 struct sk_buff *data_skb = NULL; 1518 u32 tid = GET_TID(rpl); 1519 u32 ddpvld = be32_to_cpu(rpl->ddpvld); 1520 u32 seq = be32_to_cpu(rpl->seq); 1521 u16 pdu_len_ddp = be16_to_cpu(rpl->pdu_len_ddp); 1522 1523 csk = lookup_tid(t, tid); 1524 if (unlikely(!csk)) { 1525 pr_err("can't find connection for tid %u.\n", tid); 1526 goto rel_skb; 1527 } 1528 1529 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX, 1530 "csk 0x%p,%u,0x%lx, skb 0x%p,0x%x, lhdr 0x%p, len %u, " 1531 "pdu_len_ddp %u, status %u.\n", 1532 csk, csk->state, csk->flags, skb, ddpvld, csk->skb_ulp_lhdr, 1533 ntohs(rpl->len), pdu_len_ddp, rpl->status); 1534 1535 spin_lock_bh(&csk->lock); 1536 1537 if (unlikely(csk->state >= CTP_PASSIVE_CLOSE)) { 1538 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 1539 "csk 0x%p,%u,0x%lx,%u, bad state.\n", 1540 csk, csk->state, csk->flags, csk->tid); 1541 1542 if (csk->state != CTP_ABORTING) 1543 goto abort_conn; 1544 else 1545 goto discard; 1546 } 1547 1548 cxgbi_skcb_tcp_seq(skb) = seq; 1549 cxgbi_skcb_flags(skb) = 0; 1550 cxgbi_skcb_rx_pdulen(skb) = 0; 1551 1552 skb_reset_transport_header(skb); 1553 __skb_pull(skb, sizeof(*rpl)); 1554 __pskb_trim(skb, be16_to_cpu(rpl->len)); 1555 1556 csk->rcv_nxt = seq + pdu_len_ddp; 1557 1558 if (csk->skb_ulp_lhdr) { 1559 data_skb = skb_peek(&csk->receive_queue); 1560 if (!data_skb || 1561 !cxgbi_skcb_test_flag(data_skb, SKCBF_RX_DATA)) { 1562 pr_err("Error! freelist data not found 0x%p, tid %u\n", 1563 data_skb, tid); 1564 1565 goto abort_conn; 1566 } 1567 __skb_unlink(data_skb, &csk->receive_queue); 1568 1569 cxgbi_skcb_set_flag(skb, SKCBF_RX_DATA); 1570 1571 __skb_queue_tail(&csk->receive_queue, skb); 1572 __skb_queue_tail(&csk->receive_queue, data_skb); 1573 } else { 1574 __skb_queue_tail(&csk->receive_queue, skb); 1575 } 1576 1577 csk->skb_ulp_lhdr = NULL; 1578 1579 cxgbi_skcb_set_flag(skb, SKCBF_RX_HDR); 1580 cxgbi_skcb_set_flag(skb, SKCBF_RX_STATUS); 1581 cxgbi_skcb_set_flag(skb, SKCBF_RX_ISCSI_COMPL); 1582 cxgbi_skcb_rx_ddigest(skb) = be32_to_cpu(rpl->ulp_crc); 1583 1584 cxgb4i_process_ddpvld(csk, skb, ddpvld); 1585 1586 log_debug(1 << CXGBI_DBG_PDU_RX, "csk 0x%p, skb 0x%p, f 0x%lx.\n", 1587 csk, skb, cxgbi_skcb_flags(skb)); 1588 1589 cxgbi_conn_pdu_ready(csk); 1590 spin_unlock_bh(&csk->lock); 1591 1592 return; 1593 1594abort_conn: 1595 send_abort_req(csk); 1596discard: 1597 spin_unlock_bh(&csk->lock); 1598rel_skb: 1599 __kfree_skb(skb); 1600} 1601 1602static void do_fw4_ack(struct cxgbi_device *cdev, struct sk_buff *skb) 1603{ 1604 struct cxgbi_sock *csk; 1605 struct cpl_fw4_ack *rpl = (struct cpl_fw4_ack *)skb->data; 1606 unsigned int tid = GET_TID(rpl); 1607 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); 1608 struct tid_info *t = lldi->tids; 1609 1610 csk = lookup_tid(t, tid); 1611 if (unlikely(!csk)) 1612 pr_err("can't find connection for tid %u.\n", tid); 1613 else { 1614 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 1615 "csk 0x%p,%u,0x%lx,%u.\n", 1616 csk, csk->state, csk->flags, csk->tid); 1617 cxgbi_sock_rcv_wr_ack(csk, rpl->credits, ntohl(rpl->snd_una), 1618 rpl->seq_vld); 1619 } 1620 __kfree_skb(skb); 1621} 1622 1623static void do_set_tcb_rpl(struct cxgbi_device *cdev, struct sk_buff *skb) 1624{ 1625 struct cpl_set_tcb_rpl *rpl = (struct cpl_set_tcb_rpl *)skb->data; 1626 unsigned int tid = GET_TID(rpl); 1627 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); 1628 struct tid_info *t = lldi->tids; 1629 struct cxgbi_sock *csk; 1630 1631 csk = lookup_tid(t, tid); 1632 if (!csk) { 1633 pr_err("can't find conn. for tid %u.\n", tid); 1634 return; 1635 } 1636 1637 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 1638 "csk 0x%p,%u,%lx,%u, status 0x%x.\n", 1639 csk, csk->state, csk->flags, csk->tid, rpl->status); 1640 1641 if (rpl->status != CPL_ERR_NONE) { 1642 pr_err("csk 0x%p,%u, SET_TCB_RPL status %u.\n", 1643 csk, tid, rpl->status); 1644 csk->err = -EINVAL; 1645 } 1646 1647 complete(&csk->cmpl); 1648 1649 __kfree_skb(skb); 1650} 1651 1652static int alloc_cpls(struct cxgbi_sock *csk) 1653{ 1654 csk->cpl_close = alloc_wr(sizeof(struct cpl_close_con_req), 1655 0, GFP_KERNEL); 1656 if (!csk->cpl_close) 1657 return -ENOMEM; 1658 1659 csk->cpl_abort_req = alloc_wr(sizeof(struct cpl_abort_req), 1660 0, GFP_KERNEL); 1661 if (!csk->cpl_abort_req) 1662 goto free_cpls; 1663 1664 csk->cpl_abort_rpl = alloc_wr(sizeof(struct cpl_abort_rpl), 1665 0, GFP_KERNEL); 1666 if (!csk->cpl_abort_rpl) 1667 goto free_cpls; 1668 return 0; 1669 1670free_cpls: 1671 cxgbi_sock_free_cpl_skbs(csk); 1672 return -ENOMEM; 1673} 1674 1675static inline void l2t_put(struct cxgbi_sock *csk) 1676{ 1677 if (csk->l2t) { 1678 cxgb4_l2t_release(csk->l2t); 1679 csk->l2t = NULL; 1680 cxgbi_sock_put(csk); 1681 } 1682} 1683 1684static void release_offload_resources(struct cxgbi_sock *csk) 1685{ 1686 struct cxgb4_lld_info *lldi; 1687#if IS_ENABLED(CONFIG_IPV6) 1688 struct net_device *ndev = csk->cdev->ports[csk->port_id]; 1689#endif 1690 1691 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 1692 "csk 0x%p,%u,0x%lx,%u.\n", 1693 csk, csk->state, csk->flags, csk->tid); 1694 1695 cxgbi_sock_free_cpl_skbs(csk); 1696 cxgbi_sock_purge_write_queue(csk); 1697 if (csk->wr_cred != csk->wr_max_cred) { 1698 cxgbi_sock_purge_wr_queue(csk); 1699 cxgbi_sock_reset_wr_list(csk); 1700 } 1701 1702 l2t_put(csk); 1703#if IS_ENABLED(CONFIG_IPV6) 1704 if (csk->csk_family == AF_INET6) 1705 cxgb4_clip_release(ndev, 1706 (const u32 *)&csk->saddr6.sin6_addr, 1); 1707#endif 1708 1709 if (cxgbi_sock_flag(csk, CTPF_HAS_ATID)) 1710 free_atid(csk); 1711 else if (cxgbi_sock_flag(csk, CTPF_HAS_TID)) { 1712 lldi = cxgbi_cdev_priv(csk->cdev); 1713 cxgb4_remove_tid(lldi->tids, 0, csk->tid, 1714 csk->csk_family); 1715 cxgbi_sock_clear_flag(csk, CTPF_HAS_TID); 1716 cxgbi_sock_put(csk); 1717 } 1718 csk->dst = NULL; 1719} 1720 1721#ifdef CONFIG_CHELSIO_T4_DCB 1722static inline u8 get_iscsi_dcb_state(struct net_device *ndev) 1723{ 1724 return ndev->dcbnl_ops->getstate(ndev); 1725} 1726 1727static int select_priority(int pri_mask) 1728{ 1729 if (!pri_mask) 1730 return 0; 1731 return (ffs(pri_mask) - 1); 1732} 1733 1734static u8 get_iscsi_dcb_priority(struct net_device *ndev) 1735{ 1736 int rv; 1737 u8 caps; 1738 1739 struct dcb_app iscsi_dcb_app = { 1740 .protocol = 3260 1741 }; 1742 1743 rv = (int)ndev->dcbnl_ops->getcap(ndev, DCB_CAP_ATTR_DCBX, &caps); 1744 if (rv) 1745 return 0; 1746 1747 if (caps & DCB_CAP_DCBX_VER_IEEE) { 1748 iscsi_dcb_app.selector = IEEE_8021QAZ_APP_SEL_STREAM; 1749 rv = dcb_ieee_getapp_mask(ndev, &iscsi_dcb_app); 1750 if (!rv) { 1751 iscsi_dcb_app.selector = IEEE_8021QAZ_APP_SEL_ANY; 1752 rv = dcb_ieee_getapp_mask(ndev, &iscsi_dcb_app); 1753 } 1754 } else if (caps & DCB_CAP_DCBX_VER_CEE) { 1755 iscsi_dcb_app.selector = DCB_APP_IDTYPE_PORTNUM; 1756 rv = dcb_getapp(ndev, &iscsi_dcb_app); 1757 } 1758 1759 log_debug(1 << CXGBI_DBG_ISCSI, 1760 "iSCSI priority is set to %u\n", select_priority(rv)); 1761 return select_priority(rv); 1762} 1763#endif 1764 1765static int init_act_open(struct cxgbi_sock *csk) 1766{ 1767 struct cxgbi_device *cdev = csk->cdev; 1768 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); 1769 struct net_device *ndev = cdev->ports[csk->port_id]; 1770 struct sk_buff *skb = NULL; 1771 struct neighbour *n = NULL; 1772 void *daddr; 1773 unsigned int step; 1774 unsigned int rxq_idx; 1775 unsigned int size, size6; 1776 unsigned int linkspeed; 1777 unsigned int rcv_winf, snd_winf; 1778#ifdef CONFIG_CHELSIO_T4_DCB 1779 u8 priority = 0; 1780#endif 1781 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 1782 "csk 0x%p,%u,0x%lx,%u.\n", 1783 csk, csk->state, csk->flags, csk->tid); 1784 1785 if (csk->csk_family == AF_INET) 1786 daddr = &csk->daddr.sin_addr.s_addr; 1787#if IS_ENABLED(CONFIG_IPV6) 1788 else if (csk->csk_family == AF_INET6) 1789 daddr = &csk->daddr6.sin6_addr; 1790#endif 1791 else { 1792 pr_err("address family 0x%x not supported\n", csk->csk_family); 1793 goto rel_resource; 1794 } 1795 1796 n = dst_neigh_lookup(csk->dst, daddr); 1797 1798 if (!n) { 1799 pr_err("%s, can't get neighbour of csk->dst.\n", ndev->name); 1800 goto rel_resource; 1801 } 1802 1803 if (!(n->nud_state & NUD_VALID)) 1804 neigh_event_send(n, NULL); 1805 1806 csk->atid = cxgb4_alloc_atid(lldi->tids, csk); 1807 if (csk->atid < 0) { 1808 pr_err("%s, NO atid available.\n", ndev->name); 1809 goto rel_resource_without_clip; 1810 } 1811 cxgbi_sock_set_flag(csk, CTPF_HAS_ATID); 1812 cxgbi_sock_get(csk); 1813 1814#ifdef CONFIG_CHELSIO_T4_DCB 1815 if (get_iscsi_dcb_state(ndev)) 1816 priority = get_iscsi_dcb_priority(ndev); 1817 1818 csk->dcb_priority = priority; 1819 csk->l2t = cxgb4_l2t_get(lldi->l2t, n, ndev, priority); 1820#else 1821 csk->l2t = cxgb4_l2t_get(lldi->l2t, n, ndev, 0); 1822#endif 1823 if (!csk->l2t) { 1824 pr_err("%s, cannot alloc l2t.\n", ndev->name); 1825 goto rel_resource_without_clip; 1826 } 1827 cxgbi_sock_get(csk); 1828 1829#if IS_ENABLED(CONFIG_IPV6) 1830 if (csk->csk_family == AF_INET6) 1831 cxgb4_clip_get(ndev, (const u32 *)&csk->saddr6.sin6_addr, 1); 1832#endif 1833 1834 if (is_t4(lldi->adapter_type)) { 1835 size = sizeof(struct cpl_act_open_req); 1836 size6 = sizeof(struct cpl_act_open_req6); 1837 } else if (is_t5(lldi->adapter_type)) { 1838 size = sizeof(struct cpl_t5_act_open_req); 1839 size6 = sizeof(struct cpl_t5_act_open_req6); 1840 } else { 1841 size = sizeof(struct cpl_t6_act_open_req); 1842 size6 = sizeof(struct cpl_t6_act_open_req6); 1843 } 1844 1845 if (csk->csk_family == AF_INET) 1846 skb = alloc_wr(size, 0, GFP_NOIO); 1847#if IS_ENABLED(CONFIG_IPV6) 1848 else 1849 skb = alloc_wr(size6, 0, GFP_NOIO); 1850#endif 1851 1852 if (!skb) 1853 goto rel_resource; 1854 skb->sk = (struct sock *)csk; 1855 t4_set_arp_err_handler(skb, csk, cxgbi_sock_act_open_req_arp_failure); 1856 1857 if (!csk->mtu) 1858 csk->mtu = dst_mtu(csk->dst); 1859 cxgb4_best_mtu(lldi->mtus, csk->mtu, &csk->mss_idx); 1860 csk->tx_chan = cxgb4_port_chan(ndev); 1861 csk->smac_idx = ((struct port_info *)netdev_priv(ndev))->smt_idx; 1862 step = lldi->ntxq / lldi->nchan; 1863 csk->txq_idx = cxgb4_port_idx(ndev) * step; 1864 step = lldi->nrxq / lldi->nchan; 1865 rxq_idx = (cxgb4_port_idx(ndev) * step) + (cdev->rxq_idx_cntr % step); 1866 cdev->rxq_idx_cntr++; 1867 csk->rss_qid = lldi->rxq_ids[rxq_idx]; 1868 linkspeed = ((struct port_info *)netdev_priv(ndev))->link_cfg.speed; 1869 csk->snd_win = cxgb4i_snd_win; 1870 csk->rcv_win = cxgb4i_rcv_win; 1871 if (cxgb4i_rcv_win <= 0) { 1872 csk->rcv_win = CXGB4I_DEFAULT_10G_RCV_WIN; 1873 rcv_winf = linkspeed / SPEED_10000; 1874 if (rcv_winf) 1875 csk->rcv_win *= rcv_winf; 1876 } 1877 if (cxgb4i_snd_win <= 0) { 1878 csk->snd_win = CXGB4I_DEFAULT_10G_SND_WIN; 1879 snd_winf = linkspeed / SPEED_10000; 1880 if (snd_winf) 1881 csk->snd_win *= snd_winf; 1882 } 1883 csk->wr_cred = lldi->wr_cred - 1884 DIV_ROUND_UP(sizeof(struct cpl_abort_req), 16); 1885 csk->wr_max_cred = csk->wr_cred; 1886 csk->wr_una_cred = 0; 1887 cxgbi_sock_reset_wr_list(csk); 1888 csk->err = 0; 1889 1890 pr_info_ipaddr("csk 0x%p,%u,0x%lx,%u,%u,%u, mtu %u,%u, smac %u.\n", 1891 (&csk->saddr), (&csk->daddr), csk, csk->state, 1892 csk->flags, csk->tx_chan, csk->txq_idx, csk->rss_qid, 1893 csk->mtu, csk->mss_idx, csk->smac_idx); 1894 1895 /* must wait for either a act_open_rpl or act_open_establish */ 1896 if (!try_module_get(cdev->owner)) { 1897 pr_err("%s, try_module_get failed.\n", ndev->name); 1898 goto rel_resource; 1899 } 1900 1901 cxgbi_sock_set_state(csk, CTP_ACTIVE_OPEN); 1902 if (csk->csk_family == AF_INET) 1903 send_act_open_req(csk, skb, csk->l2t); 1904#if IS_ENABLED(CONFIG_IPV6) 1905 else 1906 send_act_open_req6(csk, skb, csk->l2t); 1907#endif 1908 neigh_release(n); 1909 1910 return 0; 1911 1912rel_resource: 1913#if IS_ENABLED(CONFIG_IPV6) 1914 if (csk->csk_family == AF_INET6) 1915 cxgb4_clip_release(ndev, 1916 (const u32 *)&csk->saddr6.sin6_addr, 1); 1917#endif 1918rel_resource_without_clip: 1919 if (n) 1920 neigh_release(n); 1921 if (skb) 1922 __kfree_skb(skb); 1923 return -EINVAL; 1924} 1925 1926static cxgb4i_cplhandler_func cxgb4i_cplhandlers[NUM_CPL_CMDS] = { 1927 [CPL_ACT_ESTABLISH] = do_act_establish, 1928 [CPL_ACT_OPEN_RPL] = do_act_open_rpl, 1929 [CPL_PEER_CLOSE] = do_peer_close, 1930 [CPL_ABORT_REQ_RSS] = do_abort_req_rss, 1931 [CPL_ABORT_RPL_RSS] = do_abort_rpl_rss, 1932 [CPL_CLOSE_CON_RPL] = do_close_con_rpl, 1933 [CPL_FW4_ACK] = do_fw4_ack, 1934 [CPL_ISCSI_HDR] = do_rx_iscsi_hdr, 1935 [CPL_ISCSI_DATA] = do_rx_iscsi_data, 1936 [CPL_SET_TCB_RPL] = do_set_tcb_rpl, 1937 [CPL_RX_DATA_DDP] = do_rx_data_ddp, 1938 [CPL_RX_ISCSI_DDP] = do_rx_data_ddp, 1939 [CPL_RX_ISCSI_CMP] = do_rx_iscsi_cmp, 1940 [CPL_RX_DATA] = do_rx_data, 1941}; 1942 1943static int cxgb4i_ofld_init(struct cxgbi_device *cdev) 1944{ 1945 int rc; 1946 1947 if (cxgb4i_max_connect > CXGB4I_MAX_CONN) 1948 cxgb4i_max_connect = CXGB4I_MAX_CONN; 1949 1950 rc = cxgbi_device_portmap_create(cdev, cxgb4i_sport_base, 1951 cxgb4i_max_connect); 1952 if (rc < 0) 1953 return rc; 1954 1955 cdev->csk_release_offload_resources = release_offload_resources; 1956 cdev->csk_push_tx_frames = push_tx_frames; 1957 cdev->csk_send_abort_req = send_abort_req; 1958 cdev->csk_send_close_req = send_close_req; 1959 cdev->csk_send_rx_credits = send_rx_credits; 1960 cdev->csk_alloc_cpls = alloc_cpls; 1961 cdev->csk_init_act_open = init_act_open; 1962 1963 pr_info("cdev 0x%p, offload up, added.\n", cdev); 1964 return 0; 1965} 1966 1967static inline void 1968ulp_mem_io_set_hdr(struct cxgbi_device *cdev, 1969 struct ulp_mem_io *req, 1970 unsigned int wr_len, unsigned int dlen, 1971 unsigned int pm_addr, 1972 int tid) 1973{ 1974 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); 1975 struct ulptx_idata *idata = (struct ulptx_idata *)(req + 1); 1976 1977 INIT_ULPTX_WR(req, wr_len, 0, tid); 1978 req->wr.wr_hi = htonl(FW_WR_OP_V(FW_ULPTX_WR) | 1979 FW_WR_ATOMIC_V(0)); 1980 req->cmd = htonl(ULPTX_CMD_V(ULP_TX_MEM_WRITE) | 1981 ULP_MEMIO_ORDER_V(is_t4(lldi->adapter_type)) | 1982 T5_ULP_MEMIO_IMM_V(!is_t4(lldi->adapter_type))); 1983 req->dlen = htonl(ULP_MEMIO_DATA_LEN_V(dlen >> 5)); 1984 req->lock_addr = htonl(ULP_MEMIO_ADDR_V(pm_addr >> 5)); 1985 req->len16 = htonl(DIV_ROUND_UP(wr_len - sizeof(req->wr), 16)); 1986 1987 idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM)); 1988 idata->len = htonl(dlen); 1989} 1990 1991static struct sk_buff * 1992ddp_ppod_init_idata(struct cxgbi_device *cdev, 1993 struct cxgbi_ppm *ppm, 1994 unsigned int idx, unsigned int npods, 1995 unsigned int tid) 1996{ 1997 unsigned int pm_addr = (idx << PPOD_SIZE_SHIFT) + ppm->llimit; 1998 unsigned int dlen = npods << PPOD_SIZE_SHIFT; 1999 unsigned int wr_len = roundup(sizeof(struct ulp_mem_io) + 2000 sizeof(struct ulptx_idata) + dlen, 16); 2001 struct sk_buff *skb = alloc_wr(wr_len, 0, GFP_ATOMIC); 2002 2003 if (!skb) { 2004 pr_err("%s: %s idx %u, npods %u, OOM.\n", 2005 __func__, ppm->ndev->name, idx, npods); 2006 return NULL; 2007 } 2008 2009 ulp_mem_io_set_hdr(cdev, (struct ulp_mem_io *)skb->head, wr_len, dlen, 2010 pm_addr, tid); 2011 2012 return skb; 2013} 2014 2015static int ddp_ppod_write_idata(struct cxgbi_ppm *ppm, struct cxgbi_sock *csk, 2016 struct cxgbi_task_tag_info *ttinfo, 2017 unsigned int idx, unsigned int npods, 2018 struct scatterlist **sg_pp, 2019 unsigned int *sg_off) 2020{ 2021 struct cxgbi_device *cdev = csk->cdev; 2022 struct sk_buff *skb = ddp_ppod_init_idata(cdev, ppm, idx, npods, 2023 csk->tid); 2024 struct ulp_mem_io *req; 2025 struct ulptx_idata *idata; 2026 struct cxgbi_pagepod *ppod; 2027 int i; 2028 2029 if (!skb) 2030 return -ENOMEM; 2031 2032 req = (struct ulp_mem_io *)skb->head; 2033 idata = (struct ulptx_idata *)(req + 1); 2034 ppod = (struct cxgbi_pagepod *)(idata + 1); 2035 2036 for (i = 0; i < npods; i++, ppod++) 2037 cxgbi_ddp_set_one_ppod(ppod, ttinfo, sg_pp, sg_off); 2038 2039 cxgbi_skcb_set_flag(skb, SKCBF_TX_MEM_WRITE); 2040 cxgbi_skcb_set_flag(skb, SKCBF_TX_FLAG_COMPL); 2041 set_wr_txq(skb, CPL_PRIORITY_DATA, csk->port_id); 2042 2043 spin_lock_bh(&csk->lock); 2044 cxgbi_sock_skb_entail(csk, skb); 2045 spin_unlock_bh(&csk->lock); 2046 2047 return 0; 2048} 2049 2050static int ddp_set_map(struct cxgbi_ppm *ppm, struct cxgbi_sock *csk, 2051 struct cxgbi_task_tag_info *ttinfo) 2052{ 2053 unsigned int pidx = ttinfo->idx; 2054 unsigned int npods = ttinfo->npods; 2055 unsigned int i, cnt; 2056 int err = 0; 2057 struct scatterlist *sg = ttinfo->sgl; 2058 unsigned int offset = 0; 2059 2060 ttinfo->cid = csk->port_id; 2061 2062 for (i = 0; i < npods; i += cnt, pidx += cnt) { 2063 cnt = npods - i; 2064 2065 if (cnt > ULPMEM_IDATA_MAX_NPPODS) 2066 cnt = ULPMEM_IDATA_MAX_NPPODS; 2067 err = ddp_ppod_write_idata(ppm, csk, ttinfo, pidx, cnt, 2068 &sg, &offset); 2069 if (err < 0) 2070 break; 2071 } 2072 2073 return err; 2074} 2075 2076static int ddp_setup_conn_pgidx(struct cxgbi_sock *csk, unsigned int tid, 2077 int pg_idx) 2078{ 2079 struct sk_buff *skb; 2080 struct cpl_set_tcb_field *req; 2081 2082 if (!pg_idx || pg_idx >= DDP_PGIDX_MAX) 2083 return 0; 2084 2085 skb = alloc_wr(sizeof(*req), 0, GFP_KERNEL); 2086 if (!skb) 2087 return -ENOMEM; 2088 2089 /* set up ulp page size */ 2090 req = (struct cpl_set_tcb_field *)skb->head; 2091 INIT_TP_WR(req, csk->tid); 2092 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, csk->tid)); 2093 req->reply_ctrl = htons(NO_REPLY_V(0) | QUEUENO_V(csk->rss_qid)); 2094 req->word_cookie = htons(0); 2095 req->mask = cpu_to_be64(0x3 << 8); 2096 req->val = cpu_to_be64(pg_idx << 8); 2097 set_wr_txq(skb, CPL_PRIORITY_CONTROL, csk->port_id); 2098 2099 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 2100 "csk 0x%p, tid 0x%x, pg_idx %u.\n", csk, csk->tid, pg_idx); 2101 2102 reinit_completion(&csk->cmpl); 2103 cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb); 2104 wait_for_completion(&csk->cmpl); 2105 2106 return csk->err; 2107} 2108 2109static int ddp_setup_conn_digest(struct cxgbi_sock *csk, unsigned int tid, 2110 int hcrc, int dcrc) 2111{ 2112 struct sk_buff *skb; 2113 struct cpl_set_tcb_field *req; 2114 2115 if (!hcrc && !dcrc) 2116 return 0; 2117 2118 skb = alloc_wr(sizeof(*req), 0, GFP_KERNEL); 2119 if (!skb) 2120 return -ENOMEM; 2121 2122 csk->hcrc_len = (hcrc ? 4 : 0); 2123 csk->dcrc_len = (dcrc ? 4 : 0); 2124 /* set up ulp submode */ 2125 req = (struct cpl_set_tcb_field *)skb->head; 2126 INIT_TP_WR(req, tid); 2127 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid)); 2128 req->reply_ctrl = htons(NO_REPLY_V(0) | QUEUENO_V(csk->rss_qid)); 2129 req->word_cookie = htons(0); 2130 req->mask = cpu_to_be64(0x3 << 4); 2131 req->val = cpu_to_be64(((hcrc ? ULP_CRC_HEADER : 0) | 2132 (dcrc ? ULP_CRC_DATA : 0)) << 4); 2133 set_wr_txq(skb, CPL_PRIORITY_CONTROL, csk->port_id); 2134 2135 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 2136 "csk 0x%p, tid 0x%x, crc %d,%d.\n", csk, csk->tid, hcrc, dcrc); 2137 2138 reinit_completion(&csk->cmpl); 2139 cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb); 2140 wait_for_completion(&csk->cmpl); 2141 2142 return csk->err; 2143} 2144 2145static struct cxgbi_ppm *cdev2ppm(struct cxgbi_device *cdev) 2146{ 2147 return (struct cxgbi_ppm *)(*((struct cxgb4_lld_info *) 2148 (cxgbi_cdev_priv(cdev)))->iscsi_ppm); 2149} 2150 2151static int cxgb4i_ddp_init(struct cxgbi_device *cdev) 2152{ 2153 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); 2154 struct net_device *ndev = cdev->ports[0]; 2155 struct cxgbi_tag_format tformat; 2156 int i, err; 2157 2158 if (!lldi->vr->iscsi.size) { 2159 pr_warn("%s, iscsi NOT enabled, check config!\n", ndev->name); 2160 return -EACCES; 2161 } 2162 2163 cdev->flags |= CXGBI_FLAG_USE_PPOD_OFLDQ; 2164 2165 memset(&tformat, 0, sizeof(struct cxgbi_tag_format)); 2166 for (i = 0; i < 4; i++) 2167 tformat.pgsz_order[i] = (lldi->iscsi_pgsz_order >> (i << 3)) 2168 & 0xF; 2169 cxgbi_tagmask_check(lldi->iscsi_tagmask, &tformat); 2170 2171 pr_info("iscsi_edram.start 0x%x iscsi_edram.size 0x%x", 2172 lldi->vr->ppod_edram.start, lldi->vr->ppod_edram.size); 2173 2174 err = cxgbi_ddp_ppm_setup(lldi->iscsi_ppm, cdev, &tformat, 2175 lldi->vr->iscsi.size, lldi->iscsi_llimit, 2176 lldi->vr->iscsi.start, 2, 2177 lldi->vr->ppod_edram.start, 2178 lldi->vr->ppod_edram.size); 2179 2180 if (err < 0) 2181 return err; 2182 2183 cdev->csk_ddp_setup_digest = ddp_setup_conn_digest; 2184 cdev->csk_ddp_setup_pgidx = ddp_setup_conn_pgidx; 2185 cdev->csk_ddp_set_map = ddp_set_map; 2186 cdev->tx_max_size = min_t(unsigned int, ULP2_MAX_PDU_PAYLOAD, 2187 lldi->iscsi_iolen - ISCSI_PDU_NONPAYLOAD_LEN); 2188 cdev->rx_max_size = min_t(unsigned int, ULP2_MAX_PDU_PAYLOAD, 2189 lldi->iscsi_iolen - ISCSI_PDU_NONPAYLOAD_LEN); 2190 cdev->cdev2ppm = cdev2ppm; 2191 2192 return 0; 2193} 2194 2195static bool is_memfree(struct adapter *adap) 2196{ 2197 u32 io; 2198 2199 io = t4_read_reg(adap, MA_TARGET_MEM_ENABLE_A); 2200 if (is_t5(adap->params.chip)) { 2201 if ((io & EXT_MEM0_ENABLE_F) || (io & EXT_MEM1_ENABLE_F)) 2202 return false; 2203 } else if (io & EXT_MEM_ENABLE_F) { 2204 return false; 2205 } 2206 2207 return true; 2208} 2209 2210static void *t4_uld_add(const struct cxgb4_lld_info *lldi) 2211{ 2212 struct cxgbi_device *cdev; 2213 struct port_info *pi; 2214 struct net_device *ndev; 2215 struct adapter *adap; 2216 struct tid_info *t; 2217 u32 max_cmds = CXGB4I_SCSI_HOST_QDEPTH; 2218 u32 max_conn = CXGBI_MAX_CONN; 2219 int i, rc; 2220 2221 cdev = cxgbi_device_register(sizeof(*lldi), lldi->nports); 2222 if (!cdev) { 2223 pr_info("t4 device 0x%p, register failed.\n", lldi); 2224 return NULL; 2225 } 2226 pr_info("0x%p,0x%x, ports %u,%s, chan %u, q %u,%u, wr %u.\n", 2227 cdev, lldi->adapter_type, lldi->nports, 2228 lldi->ports[0]->name, lldi->nchan, lldi->ntxq, 2229 lldi->nrxq, lldi->wr_cred); 2230 for (i = 0; i < lldi->nrxq; i++) 2231 log_debug(1 << CXGBI_DBG_DEV, 2232 "t4 0x%p, rxq id #%d: %u.\n", 2233 cdev, i, lldi->rxq_ids[i]); 2234 2235 memcpy(cxgbi_cdev_priv(cdev), lldi, sizeof(*lldi)); 2236 cdev->flags = CXGBI_FLAG_DEV_T4; 2237 cdev->pdev = lldi->pdev; 2238 cdev->ports = lldi->ports; 2239 cdev->nports = lldi->nports; 2240 cdev->mtus = lldi->mtus; 2241 cdev->nmtus = NMTUS; 2242 cdev->rx_credit_thres = (CHELSIO_CHIP_VERSION(lldi->adapter_type) <= 2243 CHELSIO_T5) ? cxgb4i_rx_credit_thres : 0; 2244 cdev->skb_tx_rsvd = CXGB4I_TX_HEADER_LEN; 2245 cdev->skb_rx_extra = sizeof(struct cpl_iscsi_hdr); 2246 cdev->itp = &cxgb4i_iscsi_transport; 2247 cdev->owner = THIS_MODULE; 2248 2249 cdev->pfvf = FW_PFVF_CMD_PFN_V(lldi->pf); 2250 pr_info("cdev 0x%p,%s, pfvf %u.\n", 2251 cdev, lldi->ports[0]->name, cdev->pfvf); 2252 2253 rc = cxgb4i_ddp_init(cdev); 2254 if (rc) { 2255 pr_info("t4 0x%p ddp init failed %d.\n", cdev, rc); 2256 goto err_out; 2257 } 2258 2259 ndev = cdev->ports[0]; 2260 adap = netdev2adap(ndev); 2261 if (adap) { 2262 t = &adap->tids; 2263 if (t->ntids <= CXGBI_MAX_CONN) 2264 max_conn = t->ntids; 2265 2266 if (is_memfree(adap)) { 2267 cdev->flags |= CXGBI_FLAG_DEV_ISO_OFF; 2268 max_cmds = CXGB4I_SCSI_HOST_QDEPTH >> 2; 2269 2270 pr_info("%s: 0x%p, tid %u, SO adapter.\n", 2271 ndev->name, cdev, t->ntids); 2272 } 2273 } else { 2274 pr_info("%s, 0x%p, NO adapter struct.\n", ndev->name, cdev); 2275 } 2276 2277 /* ISO is enabled in T5/T6 firmware version >= 1.13.43.0 */ 2278 if (!is_t4(lldi->adapter_type) && 2279 (lldi->fw_vers >= 0x10d2b00) && 2280 !(cdev->flags & CXGBI_FLAG_DEV_ISO_OFF)) 2281 cdev->skb_iso_txhdr = sizeof(struct cpl_tx_data_iso); 2282 2283 rc = cxgb4i_ofld_init(cdev); 2284 if (rc) { 2285 pr_info("t4 0x%p ofld init failed.\n", cdev); 2286 goto err_out; 2287 } 2288 2289 cxgb4i_host_template.can_queue = max_cmds; 2290 rc = cxgbi_hbas_add(cdev, CXGB4I_MAX_LUN, max_conn, 2291 &cxgb4i_host_template, cxgb4i_stt); 2292 if (rc) 2293 goto err_out; 2294 2295 for (i = 0; i < cdev->nports; i++) { 2296 pi = netdev_priv(lldi->ports[i]); 2297 cdev->hbas[i]->port_id = pi->port_id; 2298 } 2299 return cdev; 2300 2301err_out: 2302 cxgbi_device_unregister(cdev); 2303 return ERR_PTR(-ENOMEM); 2304} 2305 2306#define RX_PULL_LEN 128 2307static int t4_uld_rx_handler(void *handle, const __be64 *rsp, 2308 const struct pkt_gl *pgl) 2309{ 2310 const struct cpl_act_establish *rpl; 2311 struct sk_buff *skb; 2312 unsigned int opc; 2313 struct cxgbi_device *cdev = handle; 2314 2315 if (pgl == NULL) { 2316 unsigned int len = 64 - sizeof(struct rsp_ctrl) - 8; 2317 2318 skb = alloc_wr(len, 0, GFP_ATOMIC); 2319 if (!skb) 2320 goto nomem; 2321 skb_copy_to_linear_data(skb, &rsp[1], len); 2322 } else { 2323 if (unlikely(*(u8 *)rsp != *(u8 *)pgl->va)) { 2324 pr_info("? FL 0x%p,RSS%#llx,FL %#llx,len %u.\n", 2325 pgl->va, be64_to_cpu(*rsp), 2326 be64_to_cpu(*(u64 *)pgl->va), 2327 pgl->tot_len); 2328 return 0; 2329 } 2330 skb = cxgb4_pktgl_to_skb(pgl, RX_PULL_LEN, RX_PULL_LEN); 2331 if (unlikely(!skb)) 2332 goto nomem; 2333 } 2334 2335 rpl = (struct cpl_act_establish *)skb->data; 2336 opc = rpl->ot.opcode; 2337 log_debug(1 << CXGBI_DBG_TOE, 2338 "cdev %p, opcode 0x%x(0x%x,0x%x), skb %p.\n", 2339 cdev, opc, rpl->ot.opcode_tid, ntohl(rpl->ot.opcode_tid), skb); 2340 if (opc >= ARRAY_SIZE(cxgb4i_cplhandlers) || !cxgb4i_cplhandlers[opc]) { 2341 pr_err("No handler for opcode 0x%x.\n", opc); 2342 __kfree_skb(skb); 2343 } else 2344 cxgb4i_cplhandlers[opc](cdev, skb); 2345 2346 return 0; 2347nomem: 2348 log_debug(1 << CXGBI_DBG_TOE, "OOM bailing out.\n"); 2349 return 1; 2350} 2351 2352static int t4_uld_state_change(void *handle, enum cxgb4_state state) 2353{ 2354 struct cxgbi_device *cdev = handle; 2355 2356 switch (state) { 2357 case CXGB4_STATE_UP: 2358 pr_info("cdev 0x%p, UP.\n", cdev); 2359 break; 2360 case CXGB4_STATE_START_RECOVERY: 2361 pr_info("cdev 0x%p, RECOVERY.\n", cdev); 2362 /* close all connections */ 2363 break; 2364 case CXGB4_STATE_DOWN: 2365 pr_info("cdev 0x%p, DOWN.\n", cdev); 2366 break; 2367 case CXGB4_STATE_DETACH: 2368 pr_info("cdev 0x%p, DETACH.\n", cdev); 2369 cxgbi_device_unregister(cdev); 2370 break; 2371 default: 2372 pr_info("cdev 0x%p, unknown state %d.\n", cdev, state); 2373 break; 2374 } 2375 return 0; 2376} 2377 2378#ifdef CONFIG_CHELSIO_T4_DCB 2379static int 2380cxgb4_dcb_change_notify(struct notifier_block *self, unsigned long val, 2381 void *data) 2382{ 2383 int i, port = 0xFF; 2384 struct net_device *ndev; 2385 struct cxgbi_device *cdev = NULL; 2386 struct dcb_app_type *iscsi_app = data; 2387 struct cxgbi_ports_map *pmap; 2388 u8 priority; 2389 2390 if (iscsi_app->dcbx & DCB_CAP_DCBX_VER_IEEE) { 2391 if ((iscsi_app->app.selector != IEEE_8021QAZ_APP_SEL_STREAM) && 2392 (iscsi_app->app.selector != IEEE_8021QAZ_APP_SEL_ANY)) 2393 return NOTIFY_DONE; 2394 2395 priority = iscsi_app->app.priority; 2396 } else if (iscsi_app->dcbx & DCB_CAP_DCBX_VER_CEE) { 2397 if (iscsi_app->app.selector != DCB_APP_IDTYPE_PORTNUM) 2398 return NOTIFY_DONE; 2399 2400 if (!iscsi_app->app.priority) 2401 return NOTIFY_DONE; 2402 2403 priority = ffs(iscsi_app->app.priority) - 1; 2404 } else { 2405 return NOTIFY_DONE; 2406 } 2407 2408 if (iscsi_app->app.protocol != 3260) 2409 return NOTIFY_DONE; 2410 2411 log_debug(1 << CXGBI_DBG_ISCSI, "iSCSI priority for ifid %d is %u\n", 2412 iscsi_app->ifindex, priority); 2413 2414 ndev = dev_get_by_index(&init_net, iscsi_app->ifindex); 2415 if (!ndev) 2416 return NOTIFY_DONE; 2417 2418 cdev = cxgbi_device_find_by_netdev_rcu(ndev, &port); 2419 2420 dev_put(ndev); 2421 if (!cdev) 2422 return NOTIFY_DONE; 2423 2424 pmap = &cdev->pmap; 2425 2426 for (i = 0; i < pmap->used; i++) { 2427 if (pmap->port_csk[i]) { 2428 struct cxgbi_sock *csk = pmap->port_csk[i]; 2429 2430 if (csk->dcb_priority != priority) { 2431 iscsi_conn_failure(csk->user_data, 2432 ISCSI_ERR_CONN_FAILED); 2433 pr_info("Restarting iSCSI connection %p with " 2434 "priority %u->%u.\n", csk, 2435 csk->dcb_priority, priority); 2436 } 2437 } 2438 } 2439 return NOTIFY_OK; 2440} 2441#endif 2442 2443static int __init cxgb4i_init_module(void) 2444{ 2445 int rc; 2446 2447 printk(KERN_INFO "%s", version); 2448 2449 rc = cxgbi_iscsi_init(&cxgb4i_iscsi_transport, &cxgb4i_stt); 2450 if (rc < 0) 2451 return rc; 2452 cxgb4_register_uld(CXGB4_ULD_ISCSI, &cxgb4i_uld_info); 2453 2454#ifdef CONFIG_CHELSIO_T4_DCB 2455 pr_info("%s dcb enabled.\n", DRV_MODULE_NAME); 2456 register_dcbevent_notifier(&cxgb4_dcb_change); 2457#endif 2458 return 0; 2459} 2460 2461static void __exit cxgb4i_exit_module(void) 2462{ 2463#ifdef CONFIG_CHELSIO_T4_DCB 2464 unregister_dcbevent_notifier(&cxgb4_dcb_change); 2465#endif 2466 cxgb4_unregister_uld(CXGB4_ULD_ISCSI); 2467 cxgbi_device_unregister_all(CXGBI_FLAG_DEV_T4); 2468 cxgbi_iscsi_cleanup(&cxgb4i_iscsi_transport, &cxgb4i_stt); 2469} 2470 2471module_init(cxgb4i_init_module); 2472module_exit(cxgb4i_exit_module); 2473