xref: /kernel/linux/linux-6.6/drivers/scsi/mpi3mr/mpi3mr_fw.c

14 mpi3mr_issue_reset(struct mpi3mr_ioc *mrioc, u16 reset_type, u32 reset_reason);
15 static int mpi3mr_setup_admin_qpair(struct mpi3mr_ioc *mrioc);
16 static void mpi3mr_process_factsdata(struct mpi3mr_ioc *mrioc,
18 static void mpi3mr_pel_wait_complete(struct mpi3mr_ioc *mrioc,
56 static void mpi3mr_sync_irqs(struct mpi3mr_ioc *mrioc)
60 	max_vectors = mrioc->intr_info_count;
63 		synchronize_irq(pci_irq_vector(mrioc->pdev, i));
66 void mpi3mr_ioc_disable_intr(struct mpi3mr_ioc *mrioc)
68 	mrioc->intr_enabled = 0;
69 	mpi3mr_sync_irqs(mrioc);
72 void mpi3mr_ioc_enable_intr(struct mpi3mr_ioc *mrioc)
74 	mrioc->intr_enabled = 1;
77 static void mpi3mr_cleanup_isr(struct mpi3mr_ioc *mrioc)
81 	mpi3mr_ioc_disable_intr(mrioc);
83 	if (!mrioc->intr_info)
86 	for (i = 0; i < mrioc->intr_info_count; i++)
87 		free_irq(pci_irq_vector(mrioc->pdev, i),
88 		    (mrioc->intr_info + i));
90 	kfree(mrioc->intr_info);
91 	mrioc->intr_info = NULL;
92 	mrioc->intr_info_count = 0;
93 	mrioc->is_intr_info_set = false;
94 	pci_free_irq_vectors(mrioc->pdev);
114 void *mpi3mr_get_reply_virt_addr(struct mpi3mr_ioc *mrioc,
120 	if ((phys_addr < mrioc->reply_buf_dma) ||
121 	    (phys_addr > mrioc->reply_buf_dma_max_address))
124 	return mrioc->reply_buf + (phys_addr - mrioc->reply_buf_dma);
127 void *mpi3mr_get_sensebuf_virt_addr(struct mpi3mr_ioc *mrioc,
133 	return mrioc->sense_buf + (phys_addr - mrioc->sense_buf_dma);
136 static void mpi3mr_repost_reply_buf(struct mpi3mr_ioc *mrioc,
142 	spin_lock_irqsave(&mrioc->reply_free_queue_lock, flags);
143 	old_idx  =  mrioc->reply_free_queue_host_index;
144 	mrioc->reply_free_queue_host_index = (
145 	    (mrioc->reply_free_queue_host_index ==
146 	    (mrioc->reply_free_qsz - 1)) ? 0 :
147 	    (mrioc->reply_free_queue_host_index + 1));
148 	mrioc->reply_free_q[old_idx] = cpu_to_le64(reply_dma);
149 	writel(mrioc->reply_free_queue_host_index,
150 	    &mrioc->sysif_regs->reply_free_host_index);
151 	spin_unlock_irqrestore(&mrioc->reply_free_queue_lock, flags);
154 void mpi3mr_repost_sense_buf(struct mpi3mr_ioc *mrioc,
160 	spin_lock_irqsave(&mrioc->sbq_lock, flags);
161 	old_idx  =  mrioc->sbq_host_index;
162 	mrioc->sbq_host_index = ((mrioc->sbq_host_index ==
163 	    (mrioc->sense_buf_q_sz - 1)) ? 0 :
164 	    (mrioc->sbq_host_index + 1));
165 	mrioc->sense_buf_q[old_idx] = cpu_to_le64(sense_buf_dma);
166 	writel(mrioc->sbq_host_index,
167 	    &mrioc->sysif_regs->sense_buffer_free_host_index);
168 	spin_unlock_irqrestore(&mrioc->sbq_lock, flags);
171 static void mpi3mr_print_event_data(struct mpi3mr_ioc *mrioc,
199 		ioc_info(mrioc, "Device Added: dev=0x%04x Form=0x%x\n",
207 		ioc_info(mrioc, "Device Info Changed: dev=0x%04x Form=0x%x\n",
215 		ioc_info(mrioc, "Device status Change: dev=0x%04x RC=0x%x\n",
223 		ioc_info(mrioc, "SAS Discovery: (%s) status (0x%08x)\n",
266 		ioc_info(mrioc, "PCIE Enumeration: (%s)",
270 			ioc_info(mrioc, "enumeration_status(0x%08x)\n",
282 	ioc_info(mrioc, "%s\n", desc);
285 static void mpi3mr_handle_events(struct mpi3mr_ioc *mrioc,
291 	mrioc->change_count = le16_to_cpu(event_reply->ioc_change_count);
292 	mpi3mr_print_event_data(mrioc, event_reply);
293 	mpi3mr_os_handle_events(mrioc, event_reply);
297 mpi3mr_get_drv_cmd(struct mpi3mr_ioc *mrioc, u16 host_tag,
304 		return &mrioc->init_cmds;
306 		return &mrioc->cfg_cmds;
308 		return &mrioc->bsg_cmds;
310 		return &mrioc->host_tm_cmds;
312 		return &mrioc->pel_abort_cmd;
314 		return &mrioc->pel_cmds;
316 		return &mrioc->transport_cmds;
320 			mpi3mr_handle_events(mrioc, def_reply);
328 		return &mrioc->dev_rmhs_cmds[idx];
334 		return &mrioc->evtack_cmds[idx];
340 static void mpi3mr_process_admin_reply_desc(struct mpi3mr_ioc *mrioc,
370 		def_reply = mpi3mr_get_reply_virt_addr(mrioc, *reply_dma);
381 			sense_buf = mpi3mr_get_sensebuf_virt_addr(mrioc,
393 	cmdptr = mpi3mr_get_drv_cmd(mrioc, host_tag, def_reply);
403 				    mrioc->reply_sz);
414 				cmdptr->callback(mrioc, cmdptr);
419 		mpi3mr_repost_sense_buf(mrioc,
423 int mpi3mr_process_admin_reply_q(struct mpi3mr_ioc *mrioc)
425 	u32 exp_phase = mrioc->admin_reply_ephase;
426 	u32 admin_reply_ci = mrioc->admin_reply_ci;
431 	if (!atomic_add_unless(&mrioc->admin_reply_q_in_use, 1, 1))
434 	reply_desc = (struct mpi3_default_reply_descriptor *)mrioc->admin_reply_base +
439 		atomic_dec(&mrioc->admin_reply_q_in_use);
444 		if (mrioc->unrecoverable)
447 		mrioc->admin_req_ci = le16_to_cpu(reply_desc->request_queue_ci);
448 		mpi3mr_process_admin_reply_desc(mrioc, reply_desc, &reply_dma);
450 			mpi3mr_repost_reply_buf(mrioc, reply_dma);
452 		if (++admin_reply_ci == mrioc->num_admin_replies) {
457 		    (struct mpi3_default_reply_descriptor *)mrioc->admin_reply_base +
464 	writel(admin_reply_ci, &mrioc->sysif_regs->admin_reply_queue_ci);
465 	mrioc->admin_reply_ci = admin_reply_ci;
466 	mrioc->admin_reply_ephase = exp_phase;
467 	atomic_dec(&mrioc->admin_reply_q_in_use);
496  * @mrioc: Adapter instance reference
506 int mpi3mr_process_op_reply_q(struct mpi3mr_ioc *mrioc,
533 		if (mrioc->unrecoverable)
537 		op_req_q = &mrioc->req_qinfo[req_q_idx];
540 		mpi3mr_process_op_reply_desc(mrioc, reply_desc, &reply_dma,
544 			mpi3mr_repost_reply_buf(mrioc, reply_dma);
562 		if (num_op_reply > mrioc->max_host_ios) {
570 	    &mrioc->sysif_regs->oper_queue_indexes[reply_qidx].consumer_index);
593 	struct mpi3mr_ioc *mrioc;
595 	mrioc = (struct mpi3mr_ioc *)shost->hostdata;
597 	if ((mrioc->reset_in_progress || mrioc->prepare_for_reset ||
598 	    mrioc->unrecoverable))
601 	num_entries = mpi3mr_process_op_reply_q(mrioc,
602 			&mrioc->op_reply_qinfo[queue_num]);
610 	struct mpi3mr_ioc *mrioc;
617 	mrioc = intr_info->mrioc;
619 	if (!mrioc->intr_enabled)
625 		num_admin_replies = mpi3mr_process_admin_reply_q(mrioc);
627 		num_op_reply = mpi3mr_process_op_reply_q(mrioc,
678 	struct mpi3mr_ioc *mrioc;
685 	mrioc = intr_info->mrioc;
690 		if (!mrioc->intr_enabled || mrioc->unrecoverable)
694 			mpi3mr_process_admin_reply_q(mrioc);
697 			    mpi3mr_process_op_reply_q(mrioc,
703 	    (num_op_reply < mrioc->max_host_ios));
715  * @mrioc: Adapter instance reference
722 static inline int mpi3mr_request_irq(struct mpi3mr_ioc *mrioc, u16 index)
724 	struct pci_dev *pdev = mrioc->pdev;
725 	struct mpi3mr_intr_info *intr_info = mrioc->intr_info + index;
728 	intr_info->mrioc = mrioc;
733 	    mrioc->driver_name, mrioc->id, index);
743 		ioc_err(mrioc, "%s: Unable to allocate interrupt %d!\n",
752 static void mpi3mr_calc_poll_queues(struct mpi3mr_ioc *mrioc, u16 max_vectors)
754 	if (!mrioc->requested_poll_qcount)
759 		(mrioc->requested_poll_qcount < max_vectors - 2)) {
760 		ioc_info(mrioc,
762 		    mrioc->requested_poll_qcount, max_vectors);
764 		ioc_info(mrioc,
766 		    mrioc->requested_poll_qcount, max_vectors);
767 		mrioc->requested_poll_qcount = 0;
773  * @mrioc: Adapter instance reference
780 static int mpi3mr_setup_isr(struct mpi3mr_ioc *mrioc, u8 setup_one)
788 	if (mrioc->is_intr_info_set)
791 	mpi3mr_cleanup_isr(mrioc);
795 		retval = pci_alloc_irq_vectors(mrioc->pdev,
798 			ioc_err(mrioc, "cannot allocate irq vectors, ret %d\n",
804 		    min_t(int, mrioc->cpu_count + 1 +
805 			mrioc->requested_poll_qcount, mrioc->msix_count);
807 		mpi3mr_calc_poll_queues(mrioc, max_vectors);
809 		ioc_info(mrioc,
811 		    mrioc->msix_count, mrioc->cpu_count);
812 		ioc_info(mrioc,
814 		    max_vectors, mrioc->requested_poll_qcount);
816 		desc.post_vectors = mrioc->requested_poll_qcount;
820 		retval = pci_alloc_irq_vectors_affinity(mrioc->pdev,
824 			ioc_err(mrioc, "cannot allocate irq vectors, ret %d\n",
835 			mrioc->op_reply_q_offset = 0;
837 			ioc_info(mrioc,
843 		mrioc->op_reply_q_offset = (max_vectors > 1) ? 1 : 0;
845 		mpi3mr_calc_poll_queues(mrioc, max_vectors);
849 	mrioc->intr_info = kzalloc(sizeof(struct mpi3mr_intr_info) * max_vectors,
851 	if (!mrioc->intr_info) {
853 		pci_free_irq_vectors(mrioc->pdev);
857 		retval = mpi3mr_request_irq(mrioc, i);
859 			mrioc->intr_info_count = i;
864 		mrioc->is_intr_info_set = true;
865 	mrioc->intr_info_count = max_vectors;
866 	mpi3mr_ioc_enable_intr(mrioc);
870 	mpi3mr_cleanup_isr(mrioc);
1001  * @mrioc: Adapter instance reference
1008 void mpi3mr_print_fault_info(struct mpi3mr_ioc *mrioc)
1012 	ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1015 		code = readl(&mrioc->sysif_regs->fault);
1016 		code1 = readl(&mrioc->sysif_regs->fault_info[0]);
1017 		code2 = readl(&mrioc->sysif_regs->fault_info[1]);
1018 		code3 = readl(&mrioc->sysif_regs->fault_info[2]);
1020 		ioc_info(mrioc,
1028  * @mrioc: Adapter instance reference
1035 enum mpi3mr_iocstate mpi3mr_get_iocstate(struct mpi3mr_ioc *mrioc)
1040 	ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1041 	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
1043 	if (mrioc->unrecoverable)
1063  * @mrioc: Adapter instance reference
1070 static inline void mpi3mr_clear_reset_history(struct mpi3mr_ioc *mrioc)
1074 	ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1076 		writel(ioc_status, &mrioc->sysif_regs->ioc_status);
1081  * @mrioc: Adapter instance reference
1089 static int mpi3mr_issue_and_process_mur(struct mpi3mr_ioc *mrioc,
1095 	ioc_info(mrioc, "Issuing Message unit Reset(MUR)\n");
1096 	if (mrioc->unrecoverable) {
1097 		ioc_info(mrioc, "IOC is unrecoverable MUR not issued\n");
1100 	mpi3mr_clear_reset_history(mrioc);
1101 	writel(reset_reason, &mrioc->sysif_regs->scratchpad[0]);
1102 	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
1104 	writel(ioc_config, &mrioc->sysif_regs->ioc_configuration);
1108 		ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1110 			mpi3mr_clear_reset_history(mrioc);
1114 			mpi3mr_print_fault_info(mrioc);
1120 	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
1126 	ioc_info(mrioc, "Base IOC Sts/Config after %s MUR is (0x%x)/(0x%x)\n",
1134  * @mrioc: Adapter instance reference
1140 mpi3mr_revalidate_factsdata(struct mpi3mr_ioc *mrioc)
1144 	if (mrioc->facts.reply_sz > mrioc->reply_sz) {
1145 		ioc_err(mrioc,
1147 		    mrioc->reply_sz, mrioc->facts.reply_sz);
1151 	if (mrioc->facts.max_op_reply_q < mrioc->num_op_reply_q) {
1152 		ioc_err(mrioc,
1154 		    mrioc->num_op_reply_q,
1155 		    mrioc->facts.max_op_reply_q);
1159 	if (mrioc->facts.max_op_req_q < mrioc->num_op_req_q) {
1160 		ioc_err(mrioc,
1162 		    mrioc->num_op_req_q, mrioc->facts.max_op_req_q);
1166 	if (mrioc->shost->max_sectors != (mrioc->facts.max_data_length / 512))
1167 		ioc_err(mrioc, "Warning: The maximum data transfer length\n"
1170 			    mrioc->shost->max_sectors * 512, mrioc->facts.max_data_length);
1172 	if ((mrioc->sas_transport_enabled) && (mrioc->facts.ioc_capabilities &
1174 		ioc_err(mrioc,
1179 	if (mrioc->facts.max_devhandle > mrioc->dev_handle_bitmap_bits) {
1180 		removepend_bitmap = bitmap_zalloc(mrioc->facts.max_devhandle,
1183 			ioc_err(mrioc,
1185 				mrioc->dev_handle_bitmap_bits,
1186 				mrioc->facts.max_devhandle);
1189 		bitmap_free(mrioc->removepend_bitmap);
1190 		mrioc->removepend_bitmap = removepend_bitmap;
1191 		ioc_info(mrioc,
1193 			 mrioc->dev_handle_bitmap_bits,
1194 			 mrioc->facts.max_devhandle);
1195 		mrioc->dev_handle_bitmap_bits = mrioc->facts.max_devhandle;
1203  * @mrioc: Adapter instance reference
1210 static int mpi3mr_bring_ioc_ready(struct mpi3mr_ioc *mrioc)
1217 	ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1218 	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
1219 	base_info = lo_hi_readq(&mrioc->sysif_regs->ioc_information);
1220 	ioc_info(mrioc, "ioc_status(0x%08x), ioc_config(0x%08x), ioc_info(0x%016llx) at the bringup\n",
1224 	mrioc->ready_timeout =
1228 	ioc_info(mrioc, "ready timeout: %d seconds\n", mrioc->ready_timeout);
1230 	ioc_state = mpi3mr_get_iocstate(mrioc);
1231 	ioc_info(mrioc, "controller is in %s state during detection\n",
1236 		timeout = mrioc->ready_timeout * 10;
1241 		if (!pci_device_is_present(mrioc->pdev)) {
1242 			mrioc->unrecoverable = 1;
1243 			ioc_err(mrioc,
1249 		ioc_state = mpi3mr_get_iocstate(mrioc);
1250 		ioc_info(mrioc,
1256 		ioc_info(mrioc, "issuing message unit reset (MUR) to bring to reset state\n");
1257 		retval = mpi3mr_issue_and_process_mur(mrioc,
1259 		ioc_state = mpi3mr_get_iocstate(mrioc);
1261 			ioc_err(mrioc,
1268 			mpi3mr_print_fault_info(mrioc);
1271 					readl(&mrioc->sysif_regs->host_diagnostic);
1275 				if (!pci_device_is_present(mrioc->pdev)) {
1276 					mrioc->unrecoverable = 1;
1277 					ioc_err(mrioc, "controller is not present at the bringup\n");
1283 		mpi3mr_print_fault_info(mrioc);
1284 		ioc_info(mrioc, "issuing soft reset to bring to reset state\n");
1285 		retval = mpi3mr_issue_reset(mrioc,
1289 			ioc_err(mrioc,
1294 	ioc_state = mpi3mr_get_iocstate(mrioc);
1296 		ioc_err(mrioc,
1301 	mpi3mr_clear_reset_history(mrioc);
1302 	retval = mpi3mr_setup_admin_qpair(mrioc);
1304 		ioc_err(mrioc, "failed to setup admin queues: error %d\n",
1309 	ioc_info(mrioc, "bringing controller to ready state\n");
1310 	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
1312 	writel(ioc_config, &mrioc->sysif_regs->ioc_configuration);
1314 	timeout = mrioc->ready_timeout * 10;
1316 		ioc_state = mpi3mr_get_iocstate(mrioc);
1318 			ioc_info(mrioc,
1323 		if (!pci_device_is_present(mrioc->pdev)) {
1324 			mrioc->unrecoverable = 1;
1325 			ioc_err(mrioc,
1334 	ioc_state = mpi3mr_get_iocstate(mrioc);
1335 	ioc_err(mrioc,
1363  * @mrioc: Adapter reference
1370 static inline bool mpi3mr_diagfault_success(struct mpi3mr_ioc *mrioc,
1377 	fault = readl(&mrioc->sysif_regs->fault) & MPI3_SYSIF_FAULT_CODE_MASK;
1379 		mpi3mr_print_fault_info(mrioc);
1387  * @mrioc: Adapter reference
1394 static inline void mpi3mr_set_diagsave(struct mpi3mr_ioc *mrioc)
1398 	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
1400 	writel(ioc_config, &mrioc->sysif_regs->ioc_configuration);
1405  * @mrioc: Adapter reference
1416 static int mpi3mr_issue_reset(struct mpi3mr_ioc *mrioc, u16 reset_type,
1427 	if (mrioc->unrecoverable)
1434 	ioc_info(mrioc, "%s reset due to %s(0x%x)\n",
1438 	mpi3mr_clear_reset_history(mrioc);
1440 		ioc_info(mrioc,
1444 			ioc_err(mrioc,
1448 			mrioc->unrecoverable = 1;
1453 		    &mrioc->sysif_regs->write_sequence);
1455 		    &mrioc->sysif_regs->write_sequence);
1457 		    &mrioc->sysif_regs->write_sequence);
1459 		    &mrioc->sysif_regs->write_sequence);
1461 		    &mrioc->sysif_regs->write_sequence);
1463 		    &mrioc->sysif_regs->write_sequence);
1465 		    &mrioc->sysif_regs->write_sequence);
1467 		host_diagnostic = readl(&mrioc->sysif_regs->host_diagnostic);
1468 		ioc_info(mrioc,
1473 	writel(reset_reason, &mrioc->sysif_regs->scratchpad[0]);
1475 	    &mrioc->sysif_regs->host_diagnostic);
1479 			ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1481 			    readl(&mrioc->sysif_regs->ioc_configuration);
1485 				mpi3mr_clear_reset_history(mrioc);
1491 		mpi3mr_print_fault_info(mrioc);
1495 			ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1496 			if (mpi3mr_diagfault_success(mrioc, ioc_status)) {
1508 	    &mrioc->sysif_regs->write_sequence);
1510 	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
1511 	ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1512 	ioc_info(mrioc,
1517 		mrioc->unrecoverable = 1;
1523  * @mrioc: Adapter reference
1534 int mpi3mr_admin_request_post(struct mpi3mr_ioc *mrioc, void *admin_req,
1542 	if (mrioc->unrecoverable) {
1543 		ioc_err(mrioc, "%s : Unrecoverable controller\n", __func__);
1547 	spin_lock_irqsave(&mrioc->admin_req_lock, flags);
1548 	areq_pi = mrioc->admin_req_pi;
1549 	areq_ci = mrioc->admin_req_ci;
1550 	max_entries = mrioc->num_admin_req;
1553 		ioc_err(mrioc, "AdminReqQ full condition detected\n");
1557 	if (!ignore_reset && mrioc->reset_in_progress) {
1558 		ioc_err(mrioc, "AdminReqQ submit reset in progress\n");
1562 	areq_entry = (u8 *)mrioc->admin_req_base +
1569 	mrioc->admin_req_pi = areq_pi;
1571 	writel(mrioc->admin_req_pi, &mrioc->sysif_regs->admin_request_queue_pi);
1574 	spin_unlock_irqrestore(&mrioc->admin_req_lock, flags);
1581  * @mrioc: Adapter instance reference
1588 static void mpi3mr_free_op_req_q_segments(struct mpi3mr_ioc *mrioc, u16 q_idx)
1594 	segments = mrioc->req_qinfo[q_idx].q_segments;
1598 	if (mrioc->enable_segqueue) {
1600 		if (mrioc->req_qinfo[q_idx].q_segment_list) {
1601 			dma_free_coherent(&mrioc->pdev->dev,
1603 			    mrioc->req_qinfo[q_idx].q_segment_list,
1604 			    mrioc->req_qinfo[q_idx].q_segment_list_dma);
1605 			mrioc->req_qinfo[q_idx].q_segment_list = NULL;
1608 		size = mrioc->req_qinfo[q_idx].segment_qd *
1609 		    mrioc->facts.op_req_sz;
1611 	for (j = 0; j < mrioc->req_qinfo[q_idx].num_segments; j++) {
1614 		dma_free_coherent(&mrioc->pdev->dev,
1618 	kfree(mrioc->req_qinfo[q_idx].q_segments);
1619 	mrioc->req_qinfo[q_idx].q_segments = NULL;
1620 	mrioc->req_qinfo[q_idx].qid = 0;
1625  * @mrioc: Adapter instance reference
1632 static void mpi3mr_free_op_reply_q_segments(struct mpi3mr_ioc *mrioc, u16 q_idx)
1638 	segments = mrioc->op_reply_qinfo[q_idx].q_segments;
1642 	if (mrioc->enable_segqueue) {
1644 		if (mrioc->op_reply_qinfo[q_idx].q_segment_list) {
1645 			dma_free_coherent(&mrioc->pdev->dev,
1647 			    mrioc->op_reply_qinfo[q_idx].q_segment_list,
1648 			    mrioc->op_reply_qinfo[q_idx].q_segment_list_dma);
1649 			mrioc->op_reply_qinfo[q_idx].q_segment_list = NULL;
1652 		size = mrioc->op_reply_qinfo[q_idx].segment_qd *
1653 		    mrioc->op_reply_desc_sz;
1655 	for (j = 0; j < mrioc->op_reply_qinfo[q_idx].num_segments; j++) {
1658 		dma_free_coherent(&mrioc->pdev->dev,
1663 	kfree(mrioc->op_reply_qinfo[q_idx].q_segments);
1664 	mrioc->op_reply_qinfo[q_idx].q_segments = NULL;
1665 	mrioc->op_reply_qinfo[q_idx].qid = 0;
1670  * @mrioc: Adapter instance reference
1678 static int mpi3mr_delete_op_reply_q(struct mpi3mr_ioc *mrioc, u16 qidx)
1681 	struct op_reply_qinfo *op_reply_q = mrioc->op_reply_qinfo + qidx;
1687 	midx = REPLY_QUEUE_IDX_TO_MSIX_IDX(qidx, mrioc->op_reply_q_offset);
1691 		ioc_err(mrioc, "Issue DelRepQ: called with invalid ReqQID\n");
1695 	(op_reply_q->qtype == MPI3MR_DEFAULT_QUEUE) ? mrioc->default_qcount-- :
1696 	    mrioc->active_poll_qcount--;
1699 	mutex_lock(&mrioc->init_cmds.mutex);
1700 	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
1702 		ioc_err(mrioc, "Issue DelRepQ: Init command is in use\n");
1703 		mutex_unlock(&mrioc->init_cmds.mutex);
1706 	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
1707 	mrioc->init_cmds.is_waiting = 1;
1708 	mrioc->init_cmds.callback = NULL;
1713 	init_completion(&mrioc->init_cmds.done);
1714 	retval = mpi3mr_admin_request_post(mrioc, &delq_req, sizeof(delq_req),
1717 		ioc_err(mrioc, "Issue DelRepQ: Admin Post failed\n");
1720 	wait_for_completion_timeout(&mrioc->init_cmds.done,
1722 	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
1723 		ioc_err(mrioc, "delete reply queue timed out\n");
1724 		mpi3mr_check_rh_fault_ioc(mrioc,
1729 	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
1731 		ioc_err(mrioc,
1733 		    (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
1734 		    mrioc->init_cmds.ioc_loginfo);
1738 	mrioc->intr_info[midx].op_reply_q = NULL;
1740 	mpi3mr_free_op_reply_q_segments(mrioc, qidx);
1742 	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
1743 	mutex_unlock(&mrioc->init_cmds.mutex);
1751  * @mrioc: Adapter instance reference
1759 static int mpi3mr_alloc_op_reply_q_segments(struct mpi3mr_ioc *mrioc, u16 qidx)
1761 	struct op_reply_qinfo *op_reply_q = mrioc->op_reply_qinfo + qidx;
1766 	if (mrioc->enable_segqueue) {
1768 		    MPI3MR_OP_REP_Q_SEG_SIZE / mrioc->op_reply_desc_sz;
1772 		op_reply_q->q_segment_list = dma_alloc_coherent(&mrioc->pdev->dev,
1780 		size = op_reply_q->num_replies * mrioc->op_reply_desc_sz;
1794 		    dma_alloc_coherent(&mrioc->pdev->dev,
1798 		if (mrioc->enable_segqueue)
1808  * @mrioc: Adapter instance reference
1816 static int mpi3mr_alloc_op_req_q_segments(struct mpi3mr_ioc *mrioc, u16 qidx)
1818 	struct op_req_qinfo *op_req_q = mrioc->req_qinfo + qidx;
1823 	if (mrioc->enable_segqueue) {
1825 		    MPI3MR_OP_REQ_Q_SEG_SIZE / mrioc->facts.op_req_sz;
1829 		op_req_q->q_segment_list = dma_alloc_coherent(&mrioc->pdev->dev,
1838 		size = op_req_q->num_requests * mrioc->facts.op_req_sz;
1852 		    dma_alloc_coherent(&mrioc->pdev->dev,
1856 		if (mrioc->enable_segqueue)
1866  * @mrioc: Adapter instance reference
1874 static int mpi3mr_create_op_reply_q(struct mpi3mr_ioc *mrioc, u16 qidx)
1877 	struct op_reply_qinfo *op_reply_q = mrioc->op_reply_qinfo + qidx;
1883 	midx = REPLY_QUEUE_IDX_TO_MSIX_IDX(qidx, mrioc->op_reply_q_offset);
1887 		ioc_err(mrioc, "CreateRepQ: called for duplicate qid %d\n",
1895 	if ((mrioc->pdev->device == MPI3_MFGPAGE_DEVID_SAS4116) &&
1896 		!mrioc->pdev->revision)
1905 		retval = mpi3mr_alloc_op_reply_q_segments(mrioc, qidx);
1907 			mpi3mr_free_op_reply_q_segments(mrioc, qidx);
1913 	mutex_lock(&mrioc->init_cmds.mutex);
1914 	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
1916 		ioc_err(mrioc, "CreateRepQ: Init command is in use\n");
1919 	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
1920 	mrioc->init_cmds.is_waiting = 1;
1921 	mrioc->init_cmds.callback = NULL;
1926 	if (midx < (mrioc->intr_info_count - mrioc->requested_poll_qcount))
1935 			cpu_to_le16(mrioc->intr_info[midx].msix_index);
1937 		create_req.msix_index = cpu_to_le16(mrioc->intr_info_count - 1);
1938 		ioc_info(mrioc, "create reply queue(polled): for qid(%d), midx(%d)\n",
1940 		if (!mrioc->active_poll_qcount)
1941 			disable_irq_nosync(pci_irq_vector(mrioc->pdev,
1942 			    mrioc->intr_info_count - 1));
1945 	if (mrioc->enable_segqueue) {
1956 	init_completion(&mrioc->init_cmds.done);
1957 	retval = mpi3mr_admin_request_post(mrioc, &create_req,
1960 		ioc_err(mrioc, "CreateRepQ: Admin Post failed\n");
1963 	wait_for_completion_timeout(&mrioc->init_cmds.done,
1965 	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
1966 		ioc_err(mrioc, "create reply queue timed out\n");
1967 		mpi3mr_check_rh_fault_ioc(mrioc,
1972 	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
1974 		ioc_err(mrioc,
1976 		    (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
1977 		    mrioc->init_cmds.ioc_loginfo);
1982 	if (midx < mrioc->intr_info_count)
1983 		mrioc->intr_info[midx].op_reply_q = op_reply_q;
1985 	(op_reply_q->qtype == MPI3MR_DEFAULT_QUEUE) ? mrioc->default_qcount++ :
1986 	    mrioc->active_poll_qcount++;
1989 	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
1990 	mutex_unlock(&mrioc->init_cmds.mutex);
1998  * @mrioc: Adapter instance reference
2007 static int mpi3mr_create_op_req_q(struct mpi3mr_ioc *mrioc, u16 idx,
2011 	struct op_req_qinfo *op_req_q = mrioc->req_qinfo + idx;
2019 		ioc_err(mrioc, "CreateReqQ: called for duplicate qid %d\n",
2033 		retval = mpi3mr_alloc_op_req_q_segments(mrioc, idx);
2035 			mpi3mr_free_op_req_q_segments(mrioc, idx);
2041 	mutex_lock(&mrioc->init_cmds.mutex);
2042 	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
2044 		ioc_err(mrioc, "CreateReqQ: Init command is in use\n");
2047 	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
2048 	mrioc->init_cmds.is_waiting = 1;
2049 	mrioc->init_cmds.callback = NULL;
2053 	if (mrioc->enable_segqueue) {
2064 	init_completion(&mrioc->init_cmds.done);
2065 	retval = mpi3mr_admin_request_post(mrioc, &create_req,
2068 		ioc_err(mrioc, "CreateReqQ: Admin Post failed\n");
2071 	wait_for_completion_timeout(&mrioc->init_cmds.done,
2073 	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
2074 		ioc_err(mrioc, "create request queue timed out\n");
2075 		mpi3mr_check_rh_fault_ioc(mrioc,
2080 	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
2082 		ioc_err(mrioc,
2084 		    (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
2085 		    mrioc->init_cmds.ioc_loginfo);
2092 	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
2093 	mutex_unlock(&mrioc->init_cmds.mutex);
2101  * @mrioc: Adapter instance reference
2108 static int mpi3mr_create_op_queues(struct mpi3mr_ioc *mrioc)
2113 	num_queues = min_t(int, mrioc->facts.max_op_reply_q,
2114 	    mrioc->facts.max_op_req_q);
2117 	    mrioc->intr_info_count - mrioc->op_reply_q_offset;
2118 	if (!mrioc->num_queues)
2119 		mrioc->num_queues = min_t(int, num_queues, msix_count_op_q);
2124 	num_queues = mrioc->num_op_reply_q ?
2125 	    mrioc->num_op_reply_q : mrioc->num_queues;
2126 	ioc_info(mrioc, "trying to create %d operational queue pairs\n",
2129 	if (!mrioc->req_qinfo) {
2130 		mrioc->req_qinfo = kcalloc(num_queues,
2132 		if (!mrioc->req_qinfo) {
2137 		mrioc->op_reply_qinfo = kzalloc(sizeof(struct op_reply_qinfo) *
2139 		if (!mrioc->op_reply_qinfo) {
2145 	if (mrioc->enable_segqueue)
2146 		ioc_info(mrioc,
2150 		if (mpi3mr_create_op_reply_q(mrioc, i)) {
2151 			ioc_err(mrioc, "Cannot create OP RepQ %d\n", i);
2154 		if (mpi3mr_create_op_req_q(mrioc, i,
2155 		    mrioc->op_reply_qinfo[i].qid)) {
2156 			ioc_err(mrioc, "Cannot create OP ReqQ %d\n", i);
2157 			mpi3mr_delete_op_reply_q(mrioc, i);
2167 	mrioc->num_op_reply_q = mrioc->num_op_req_q = i;
2168 	ioc_info(mrioc,
2170 	    mrioc->num_op_reply_q, mrioc->default_qcount,
2171 	    mrioc->active_poll_qcount);
2175 	kfree(mrioc->req_qinfo);
2176 	mrioc->req_qinfo = NULL;
2178 	kfree(mrioc->op_reply_qinfo);
2179 	mrioc->op_reply_qinfo = NULL;
2186  * @mrioc: Adapter reference
2196 int mpi3mr_op_request_post(struct mpi3mr_ioc *mrioc,
2204 	u16 req_sz = mrioc->facts.op_req_sz;
2209 	if (mrioc->unrecoverable)
2218 		    reply_qidx, mrioc->op_reply_q_offset);
2219 		mpi3mr_process_op_reply_q(mrioc, mrioc->intr_info[midx].op_reply_q);
2227 	if (mrioc->reset_in_progress) {
2228 		ioc_err(mrioc, "OpReqQ submit reset in progress\n");
2245 	if (atomic_inc_return(&mrioc->op_reply_qinfo[reply_qidx].pend_ios)
2247 		mrioc->op_reply_qinfo[reply_qidx].enable_irq_poll = true;
2249 	atomic_inc_return(&mrioc->op_reply_qinfo[reply_qidx].pend_ios);
2253 	    &mrioc->sysif_regs->oper_queue_indexes[reply_qidx].producer_index);
2263  * @mrioc: Adapter instance reference
2274 void mpi3mr_check_rh_fault_ioc(struct mpi3mr_ioc *mrioc, u32 reason_code)
2278 	if (mrioc->unrecoverable) {
2279 		ioc_err(mrioc, "controller is unrecoverable\n");
2283 	if (!pci_device_is_present(mrioc->pdev)) {
2284 		mrioc->unrecoverable = 1;
2285 		ioc_err(mrioc, "controller is not present\n");
2289 	ioc_status = readl(&mrioc->sysif_regs->ioc_status);
2292 		mpi3mr_print_fault_info(mrioc);
2295 	mpi3mr_set_diagsave(mrioc);
2296 	mpi3mr_issue_reset(mrioc, MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,
2300 		host_diagnostic = readl(&mrioc->sysif_regs->host_diagnostic);
2309  * @mrioc: Adapter reference
2316 static int mpi3mr_sync_timestamp(struct mpi3mr_ioc *mrioc)
2323 	mutex_lock(&mrioc->init_cmds.mutex);
2324 	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
2326 		ioc_err(mrioc, "Issue IOUCTL time_stamp: command is in use\n");
2327 		mutex_unlock(&mrioc->init_cmds.mutex);
2330 	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
2331 	mrioc->init_cmds.is_waiting = 1;
2332 	mrioc->init_cmds.callback = NULL;
2339 	init_completion(&mrioc->init_cmds.done);
2340 	retval = mpi3mr_admin_request_post(mrioc, &iou_ctrl,
2343 		ioc_err(mrioc, "Issue IOUCTL time_stamp: Admin Post failed\n");
2347 	wait_for_completion_timeout(&mrioc->init_cmds.done,
2349 	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
2350 		ioc_err(mrioc, "Issue IOUCTL time_stamp: command timed out\n");
2351 		mrioc->init_cmds.is_waiting = 0;
2352 		if (!(mrioc->init_cmds.state & MPI3MR_CMD_RESET))
2353 			mpi3mr_check_rh_fault_ioc(mrioc,
2358 	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
2360 		ioc_err(mrioc,
2362 		    (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
2363 		    mrioc->init_cmds.ioc_loginfo);
2369 	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
2370 	mutex_unlock(&mrioc->init_cmds.mutex);
2378  * @mrioc: Adapter reference
2385 static int mpi3mr_print_pkg_ver(struct mpi3mr_ioc *mrioc)
2395 	data = dma_alloc_coherent(&mrioc->pdev->dev, data_len, &data_dma,
2401 	mutex_lock(&mrioc->init_cmds.mutex);
2402 	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
2403 		ioc_err(mrioc, "sending get package version failed due to command in use\n");
2404 		mutex_unlock(&mrioc->init_cmds.mutex);
2407 	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
2408 	mrioc->init_cmds.is_waiting = 1;
2409 	mrioc->init_cmds.callback = NULL;
2419 	init_completion(&mrioc->init_cmds.done);
2420 	retval = mpi3mr_admin_request_post(mrioc, &ci_upload,
2423 		ioc_err(mrioc, "posting get package version failed\n");
2426 	wait_for_completion_timeout(&mrioc->init_cmds.done,
2428 	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
2429 		ioc_err(mrioc, "get package version timed out\n");
2430 		mpi3mr_check_rh_fault_ioc(mrioc,
2435 	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
2439 			ioc_info(mrioc,
2451 	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
2452 	mutex_unlock(&mrioc->init_cmds.mutex);
2456 		dma_free_coherent(&mrioc->pdev->dev, data_len, data,
2473 	struct mpi3mr_ioc *mrioc =
2480 	if (mrioc->reset_in_progress)
2483 	if (!mrioc->unrecoverable && !pci_device_is_present(mrioc->pdev)) {
2484 		ioc_err(mrioc, "watchdog could not detect the controller\n");
2485 		mrioc->unrecoverable = 1;
2488 	if (mrioc->unrecoverable) {
2489 		ioc_err(mrioc,
2491 		mpi3mr_flush_cmds_for_unrecovered_controller(mrioc);
2495 	if (mrioc->ts_update_counter++ >= MPI3MR_TSUPDATE_INTERVAL) {
2496 		mrioc->ts_update_counter = 0;
2497 		mpi3mr_sync_timestamp(mrioc);
2500 	if ((mrioc->prepare_for_reset) &&
2501 	    ((mrioc->prepare_for_reset_timeout_counter++) >=
2503 		mpi3mr_soft_reset_handler(mrioc,
2508 	ioc_status = readl(&mrioc->sysif_regs->ioc_status);
2510 		mpi3mr_soft_reset_handler(mrioc, MPI3MR_RESET_FROM_FIRMWARE, 0);
2515 	ioc_state = mpi3mr_get_iocstate(mrioc);
2519 	fault = readl(&mrioc->sysif_regs->fault) & MPI3_SYSIF_FAULT_CODE_MASK;
2520 	host_diagnostic = readl(&mrioc->sysif_regs->host_diagnostic);
2522 		if (!mrioc->diagsave_timeout) {
2523 			mpi3mr_print_fault_info(mrioc);
2524 			ioc_warn(mrioc, "diag save in progress\n");
2526 		if ((mrioc->diagsave_timeout++) <= MPI3_SYSIF_DIAG_SAVE_TIMEOUT)
2530 	mpi3mr_print_fault_info(mrioc);
2531 	mrioc->diagsave_timeout = 0;
2536 		ioc_warn(mrioc,
2538 		mrioc->unrecoverable = 1;
2548 	mpi3mr_soft_reset_handler(mrioc, reset_reason, 0);
2552 	spin_lock_irqsave(&mrioc->watchdog_lock, flags);
2553 	if (mrioc->watchdog_work_q)
2554 		queue_delayed_work(mrioc->watchdog_work_q,
2555 		    &mrioc->watchdog_work,
2557 	spin_unlock_irqrestore(&mrioc->watchdog_lock, flags);
2563  * @mrioc: Adapter instance reference
2570 void mpi3mr_start_watchdog(struct mpi3mr_ioc *mrioc)
2572 	if (mrioc->watchdog_work_q)
2575 	INIT_DELAYED_WORK(&mrioc->watchdog_work, mpi3mr_watchdog_work);
2576 	snprintf(mrioc->watchdog_work_q_name,
2577 	    sizeof(mrioc->watchdog_work_q_name), "watchdog_%s%d", mrioc->name,
2578 	    mrioc->id);
2579 	mrioc->watchdog_work_q =
2580 	    create_singlethread_workqueue(mrioc->watchdog_work_q_name);
2581 	if (!mrioc->watchdog_work_q) {
2582 		ioc_err(mrioc, "%s: failed (line=%d)\n", __func__, __LINE__);
2586 	if (mrioc->watchdog_work_q)
2587 		queue_delayed_work(mrioc->watchdog_work_q,
2588 		    &mrioc->watchdog_work,
2594  * @mrioc: Adapter instance reference
2601 void mpi3mr_stop_watchdog(struct mpi3mr_ioc *mrioc)
2606 	spin_lock_irqsave(&mrioc->watchdog_lock, flags);
2607 	wq = mrioc->watchdog_work_q;
2608 	mrioc->watchdog_work_q = NULL;
2609 	spin_unlock_irqrestore(&mrioc->watchdog_lock, flags);
2611 		if (!cancel_delayed_work_sync(&mrioc->watchdog_work))
2619  * @mrioc: Adapter instance reference
2626 static int mpi3mr_setup_admin_qpair(struct mpi3mr_ioc *mrioc)
2631 	mrioc->admin_req_q_sz = MPI3MR_ADMIN_REQ_Q_SIZE;
2632 	mrioc->num_admin_req = mrioc->admin_req_q_sz /
2634 	mrioc->admin_req_ci = mrioc->admin_req_pi = 0;
2636 	mrioc->admin_reply_q_sz = MPI3MR_ADMIN_REPLY_Q_SIZE;
2637 	mrioc->num_admin_replies = mrioc->admin_reply_q_sz /
2639 	mrioc->admin_reply_ci = 0;
2640 	mrioc->admin_reply_ephase = 1;
2641 	atomic_set(&mrioc->admin_reply_q_in_use, 0);
2643 	if (!mrioc->admin_req_base) {
2644 		mrioc->admin_req_base = dma_alloc_coherent(&mrioc->pdev->dev,
2645 		    mrioc->admin_req_q_sz, &mrioc->admin_req_dma, GFP_KERNEL);
2647 		if (!mrioc->admin_req_base) {
2652 		mrioc->admin_reply_base = dma_alloc_coherent(&mrioc->pdev->dev,
2653 		    mrioc->admin_reply_q_sz, &mrioc->admin_reply_dma,
2656 		if (!mrioc->admin_reply_base) {
2662 	num_admin_entries = (mrioc->num_admin_replies << 16) |
2663 	    (mrioc->num_admin_req);
2664 	writel(num_admin_entries, &mrioc->sysif_regs->admin_queue_num_entries);
2665 	mpi3mr_writeq(mrioc->admin_req_dma,
2666 	    &mrioc->sysif_regs->admin_request_queue_address);
2667 	mpi3mr_writeq(mrioc->admin_reply_dma,
2668 	    &mrioc->sysif_regs->admin_reply_queue_address);
2669 	writel(mrioc->admin_req_pi, &mrioc->sysif_regs->admin_request_queue_pi);
2670 	writel(mrioc->admin_reply_ci, &mrioc->sysif_regs->admin_reply_queue_ci);
2675 	if (mrioc->admin_reply_base) {
2676 		dma_free_coherent(&mrioc->pdev->dev, mrioc->admin_reply_q_sz,
2677 		    mrioc->admin_reply_base, mrioc->admin_reply_dma);
2678 		mrioc->admin_reply_base = NULL;
2680 	if (mrioc->admin_req_base) {
2681 		dma_free_coherent(&mrioc->pdev->dev, mrioc->admin_req_q_sz,
2682 		    mrioc->admin_req_base, mrioc->admin_req_dma);
2683 		mrioc->admin_req_base = NULL;
2690  * @mrioc: Adapter instance reference
2698 static int mpi3mr_issue_iocfacts(struct mpi3mr_ioc *mrioc,
2708 	data = dma_alloc_coherent(&mrioc->pdev->dev, data_len, &data_dma,
2717 	mutex_lock(&mrioc->init_cmds.mutex);
2718 	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
2720 		ioc_err(mrioc, "Issue IOCFacts: Init command is in use\n");
2721 		mutex_unlock(&mrioc->init_cmds.mutex);
2724 	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
2725 	mrioc->init_cmds.is_waiting = 1;
2726 	mrioc->init_cmds.callback = NULL;
2733 	init_completion(&mrioc->init_cmds.done);
2734 	retval = mpi3mr_admin_request_post(mrioc, &iocfacts_req,
2737 		ioc_err(mrioc, "Issue IOCFacts: Admin Post failed\n");
2740 	wait_for_completion_timeout(&mrioc->init_cmds.done,
2742 	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
2743 		ioc_err(mrioc, "ioc_facts timed out\n");
2744 		mpi3mr_check_rh_fault_ioc(mrioc,
2749 	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
2751 		ioc_err(mrioc,
2753 		    (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
2754 		    mrioc->init_cmds.ioc_loginfo);
2759 	mpi3mr_process_factsdata(mrioc, facts_data);
2761 	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
2762 	mutex_unlock(&mrioc->init_cmds.mutex);
2766 		dma_free_coherent(&mrioc->pdev->dev, data_len, data, data_dma);
2773  * @mrioc: Adapter instance reference
2780 static inline int mpi3mr_check_reset_dma_mask(struct mpi3mr_ioc *mrioc)
2782 	struct pci_dev *pdev = mrioc->pdev;
2784 	u64 facts_dma_mask = DMA_BIT_MASK(mrioc->facts.dma_mask);
2786 	if (!mrioc->facts.dma_mask || (mrioc->dma_mask <= facts_dma_mask))
2789 	ioc_info(mrioc, "Changing DMA mask from 0x%016llx to 0x%016llx\n",
2790 	    mrioc->dma_mask, facts_dma_mask);
2794 		ioc_err(mrioc, "Setting DMA mask to 0x%016llx failed: %d\n",
2798 	mrioc->dma_mask = facts_dma_mask;
2804  * @mrioc: Adapter instance reference
2812 static void mpi3mr_process_factsdata(struct mpi3mr_ioc *mrioc,
2819 		ioc_warn(mrioc,
2825 	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
2829 		ioc_err(mrioc,
2834 	memset(&mrioc->facts, 0, sizeof(mrioc->facts));
2837 	mrioc->facts.op_req_sz = req_sz;
2838 	mrioc->op_reply_desc_sz = 1 << ((ioc_config &
2842 	mrioc->facts.ioc_num = facts_data->ioc_number;
2843 	mrioc->facts.who_init = facts_data->who_init;
2844 	mrioc->facts.max_msix_vectors = le16_to_cpu(facts_data->max_msix_vectors);
2845 	mrioc->facts.personality = (facts_flags &
2847 	mrioc->facts.dma_mask = (facts_flags &
2850 	mrioc->facts.protocol_flags = facts_data->protocol_flags;
2851 	mrioc->facts.mpi_version = le32_to_cpu(facts_data->mpi_version.word);
2852 	mrioc->facts.max_reqs = le16_to_cpu(facts_data->max_outstanding_requests);
2853 	mrioc->facts.product_id = le16_to_cpu(facts_data->product_id);
2854 	mrioc->facts.reply_sz = le16_to_cpu(facts_data->reply_frame_size) * 4;
2855 	mrioc->facts.exceptions = le16_to_cpu(facts_data->ioc_exceptions);
2856 	mrioc->facts.max_perids = le16_to_cpu(facts_data->max_persistent_id);
2857 	mrioc->facts.max_vds = le16_to_cpu(facts_data->max_vds);
2858 	mrioc->facts.max_hpds = le16_to_cpu(facts_data->max_host_pds);
2859 	mrioc->facts.max_advhpds = le16_to_cpu(facts_data->max_adv_host_pds);
2860 	mrioc->facts.max_raid_pds = le16_to_cpu(facts_data->max_raid_pds);
2861 	mrioc->facts.max_nvme = le16_to_cpu(facts_data->max_nvme);
2862 	mrioc->facts.max_pcie_switches =
2864 	mrioc->facts.max_sasexpanders =
2866 	mrioc->facts.max_data_length = le16_to_cpu(facts_data->max_data_length);
2867 	mrioc->facts.max_sasinitiators =
2869 	mrioc->facts.max_enclosures = le16_to_cpu(facts_data->max_enclosures);
2870 	mrioc->facts.min_devhandle = le16_to_cpu(facts_data->min_dev_handle);
2871 	mrioc->facts.max_devhandle = le16_to_cpu(facts_data->max_dev_handle);
2872 	mrioc->facts.max_op_req_q =
2874 	mrioc->facts.max_op_reply_q =
2876 	mrioc->facts.ioc_capabilities =
2878 	mrioc->facts.fw_ver.build_num =
2880 	mrioc->facts.fw_ver.cust_id =
2882 	mrioc->facts.fw_ver.ph_minor = facts_data->fw_version.phase_minor;
2883 	mrioc->facts.fw_ver.ph_major = facts_data->fw_version.phase_major;
2884 	mrioc->facts.fw_ver.gen_minor = facts_data->fw_version.gen_minor;
2885 	mrioc->facts.fw_ver.gen_major = facts_data->fw_version.gen_major;
2886 	mrioc->msix_count = min_t(int, mrioc->msix_count,
2887 	    mrioc->facts.max_msix_vectors);
2888 	mrioc->facts.sge_mod_mask = facts_data->sge_modifier_mask;
2889 	mrioc->facts.sge_mod_value = facts_data->sge_modifier_value;
2890 	mrioc->facts.sge_mod_shift = facts_data->sge_modifier_shift;
2891 	mrioc->facts.shutdown_timeout =
2894 	mrioc->facts.max_dev_per_tg =
2896 	mrioc->facts.io_throttle_data_length =
2898 	mrioc->facts.max_io_throttle_group =
2900 	mrioc->facts.io_throttle_low = le16_to_cpu(facts_data->io_throttle_low);
2901 	mrioc->facts.io_throttle_high =
2904 	if (mrioc->facts.max_data_length ==
2906 		mrioc->facts.max_data_length = MPI3MR_DEFAULT_MAX_IO_SIZE;
2908 		mrioc->facts.max_data_length *= MPI3MR_PAGE_SIZE_4K;
2910 	if (mrioc->facts.io_throttle_data_length)
2911 		mrioc->io_throttle_data_length =
2912 		    (mrioc->facts.io_throttle_data_length * 2 * 4);
2915 		mrioc->io_throttle_data_length = (mrioc->facts.max_data_length / 512) + 2;
2917 	mrioc->io_throttle_high = (mrioc->facts.io_throttle_high * 2 * 1024);
2918 	mrioc->io_throttle_low = (mrioc->facts.io_throttle_low * 2 * 1024);
2920 	ioc_info(mrioc, "ioc_num(%d), maxopQ(%d), maxopRepQ(%d), maxdh(%d),",
2921 	    mrioc->facts.ioc_num, mrioc->facts.max_op_req_q,
2922 	    mrioc->facts.max_op_reply_q, mrioc->facts.max_devhandle);
2923 	ioc_info(mrioc,
2925 	    mrioc->facts.max_reqs, mrioc->facts.min_devhandle,
2926 	    mrioc->facts.max_msix_vectors, mrioc->facts.max_perids);
2927 	ioc_info(mrioc, "SGEModMask 0x%x SGEModVal 0x%x SGEModShift 0x%x ",
2928 	    mrioc->facts.sge_mod_mask, mrioc->facts.sge_mod_value,
2929 	    mrioc->facts.sge_mod_shift);
2930 	ioc_info(mrioc, "DMA mask %d InitialPE status 0x%x max_data_len (%d)\n",
2931 	    mrioc->facts.dma_mask, (facts_flags &
2932 	    MPI3_IOCFACTS_FLAGS_INITIAL_PORT_ENABLE_MASK), mrioc->facts.max_data_length);
2933 	ioc_info(mrioc,
2935 	    mrioc->facts.max_dev_per_tg, mrioc->facts.max_io_throttle_group);
2936 	ioc_info(mrioc,
2938 	   mrioc->facts.io_throttle_data_length * 4,
2939 	   mrioc->facts.io_throttle_high, mrioc->facts.io_throttle_low);
2944  * @mrioc: Adapter instance reference
2951 static int mpi3mr_alloc_reply_sense_bufs(struct mpi3mr_ioc *mrioc)
2956 	if (mrioc->init_cmds.reply)
2959 	mrioc->init_cmds.reply = kzalloc(mrioc->reply_sz, GFP_KERNEL);
2960 	if (!mrioc->init_cmds.reply)
2963 	mrioc->bsg_cmds.reply = kzalloc(mrioc->reply_sz, GFP_KERNEL);
2964 	if (!mrioc->bsg_cmds.reply)
2967 	mrioc->transport_cmds.reply = kzalloc(mrioc->reply_sz, GFP_KERNEL);
2968 	if (!mrioc->transport_cmds.reply)
2972 		mrioc->dev_rmhs_cmds[i].reply = kzalloc(mrioc->reply_sz,
2974 		if (!mrioc->dev_rmhs_cmds[i].reply)
2979 		mrioc->evtack_cmds[i].reply = kzalloc(mrioc->reply_sz,
2981 		if (!mrioc->evtack_cmds[i].reply)
2985 	mrioc->host_tm_cmds.reply = kzalloc(mrioc->reply_sz, GFP_KERNEL);
2986 	if (!mrioc->host_tm_cmds.reply)
2989 	mrioc->pel_cmds.reply = kzalloc(mrioc->reply_sz, GFP_KERNEL);
2990 	if (!mrioc->pel_cmds.reply)
2993 	mrioc->pel_abort_cmd.reply = kzalloc(mrioc->reply_sz, GFP_KERNEL);
2994 	if (!mrioc->pel_abort_cmd.reply)
2997 	mrioc->dev_handle_bitmap_bits = mrioc->facts.max_devhandle;
2998 	mrioc->removepend_bitmap = bitmap_zalloc(mrioc->dev_handle_bitmap_bits,
3000 	if (!mrioc->removepend_bitmap)
3003 	mrioc->devrem_bitmap = bitmap_zalloc(MPI3MR_NUM_DEVRMCMD, GFP_KERNEL);
3004 	if (!mrioc->devrem_bitmap)
3007 	mrioc->evtack_cmds_bitmap = bitmap_zalloc(MPI3MR_NUM_EVTACKCMD,
3009 	if (!mrioc->evtack_cmds_bitmap)
3012 	mrioc->num_reply_bufs = mrioc->facts.max_reqs + MPI3MR_NUM_EVT_REPLIES;
3013 	mrioc->reply_free_qsz = mrioc->num_reply_bufs + 1;
3014 	mrioc->num_sense_bufs = mrioc->facts.max_reqs / MPI3MR_SENSEBUF_FACTOR;
3015 	mrioc->sense_buf_q_sz = mrioc->num_sense_bufs + 1;
3018 	sz = mrioc->num_reply_bufs * mrioc->reply_sz;
3019 	mrioc->reply_buf_pool = dma_pool_create("reply_buf pool",
3020 	    &mrioc->pdev->dev, sz, 16, 0);
3021 	if (!mrioc->reply_buf_pool) {
3022 		ioc_err(mrioc, "reply buf pool: dma_pool_create failed\n");
3026 	mrioc->reply_buf = dma_pool_zalloc(mrioc->reply_buf_pool, GFP_KERNEL,
3027 	    &mrioc->reply_buf_dma);
3028 	if (!mrioc->reply_buf)
3031 	mrioc->reply_buf_dma_max_address = mrioc->reply_buf_dma + sz;
3034 	sz = mrioc->reply_free_qsz * 8;
3035 	mrioc->reply_free_q_pool = dma_pool_create("reply_free_q pool",
3036 	    &mrioc->pdev->dev, sz, 8, 0);
3037 	if (!mrioc->reply_free_q_pool) {
3038 		ioc_err(mrioc, "reply_free_q pool: dma_pool_create failed\n");
3041 	mrioc->reply_free_q = dma_pool_zalloc(mrioc->reply_free_q_pool,
3042 	    GFP_KERNEL, &mrioc->reply_free_q_dma);
3043 	if (!mrioc->reply_free_q)
3047 	sz = mrioc->num_sense_bufs * MPI3MR_SENSE_BUF_SZ;
3048 	mrioc->sense_buf_pool = dma_pool_create("sense_buf pool",
3049 	    &mrioc->pdev->dev, sz, 4, 0);
3050 	if (!mrioc->sense_buf_pool) {
3051 		ioc_err(mrioc, "sense_buf pool: dma_pool_create failed\n");
3054 	mrioc->sense_buf = dma_pool_zalloc(mrioc->sense_buf_pool, GFP_KERNEL,
3055 	    &mrioc->sense_buf_dma);
3056 	if (!mrioc->sense_buf)
3060 	sz = mrioc->sense_buf_q_sz * 8;
3061 	mrioc->sense_buf_q_pool = dma_pool_create("sense_buf_q pool",
3062 	    &mrioc->pdev->dev, sz, 8, 0);
3063 	if (!mrioc->sense_buf_q_pool) {
3064 		ioc_err(mrioc, "sense_buf_q pool: dma_pool_create failed\n");
3067 	mrioc->sense_buf_q = dma_pool_zalloc(mrioc->sense_buf_q_pool,
3068 	    GFP_KERNEL, &mrioc->sense_buf_q_dma);
3069 	if (!mrioc->sense_buf_q)
3082  * @mrioc: Adapter instance reference
3089 static void mpimr_initialize_reply_sbuf_queues(struct mpi3mr_ioc *mrioc)
3094 	sz = mrioc->num_reply_bufs * mrioc->reply_sz;
3095 	ioc_info(mrioc,
3097 	    mrioc->reply_buf, mrioc->num_reply_bufs, mrioc->reply_sz,
3098 	    (sz / 1024), (unsigned long long)mrioc->reply_buf_dma);
3099 	sz = mrioc->reply_free_qsz * 8;
3100 	ioc_info(mrioc,
3102 	    mrioc->reply_free_q, mrioc->reply_free_qsz, 8, (sz / 1024),
3103 	    (unsigned long long)mrioc->reply_free_q_dma);
3104 	sz = mrioc->num_sense_bufs * MPI3MR_SENSE_BUF_SZ;
3105 	ioc_info(mrioc,
3107 	    mrioc->sense_buf, mrioc->num_sense_bufs, MPI3MR_SENSE_BUF_SZ,
3108 	    (sz / 1024), (unsigned long long)mrioc->sense_buf_dma);
3109 	sz = mrioc->sense_buf_q_sz * 8;
3110 	ioc_info(mrioc,
3112 	    mrioc->sense_buf_q, mrioc->sense_buf_q_sz, 8, (sz / 1024),
3113 	    (unsigned long long)mrioc->sense_buf_q_dma);
3116 	for (i = 0, phy_addr = mrioc->reply_buf_dma;
3117 	    i < mrioc->num_reply_bufs; i++, phy_addr += mrioc->reply_sz)
3118 		mrioc->reply_free_q[i] = cpu_to_le64(phy_addr);
3119 	mrioc->reply_free_q[i] = cpu_to_le64(0);
3122 	for (i = 0, phy_addr = mrioc->sense_buf_dma;
3123 	    i < mrioc->num_sense_bufs; i++, phy_addr += MPI3MR_SENSE_BUF_SZ)
3124 		mrioc->sense_buf_q[i] = cpu_to_le64(phy_addr);
3125 	mrioc->sense_buf_q[i] = cpu_to_le64(0);
3130  * @mrioc: Adapter instance reference
3137 static int mpi3mr_issue_iocinit(struct mpi3mr_ioc *mrioc)
3146 	drv_info = dma_alloc_coherent(&mrioc->pdev->dev, data_len, &data_dma,
3152 	mpimr_initialize_reply_sbuf_queues(mrioc);
3163 	memcpy((u8 *)&mrioc->driver_info, (u8 *)drv_info,
3164 	    sizeof(mrioc->driver_info));
3167 	mutex_lock(&mrioc->init_cmds.mutex);
3168 	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
3170 		ioc_err(mrioc, "Issue IOCInit: Init command is in use\n");
3171 		mutex_unlock(&mrioc->init_cmds.mutex);
3174 	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
3175 	mrioc->init_cmds.is_waiting = 1;
3176 	mrioc->init_cmds.callback = NULL;
3184 	iocinit_req.reply_free_queue_depth = cpu_to_le16(mrioc->reply_free_qsz);
3186 	    cpu_to_le64(mrioc->reply_free_q_dma);
3189 	    cpu_to_le16(mrioc->sense_buf_q_sz);
3191 	    cpu_to_le64(mrioc->sense_buf_q_dma);
3197 	init_completion(&mrioc->init_cmds.done);
3198 	retval = mpi3mr_admin_request_post(mrioc, &iocinit_req,
3201 		ioc_err(mrioc, "Issue IOCInit: Admin Post failed\n");
3204 	wait_for_completion_timeout(&mrioc->init_cmds.done,
3206 	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
3207 		mpi3mr_check_rh_fault_ioc(mrioc,
3209 		ioc_err(mrioc, "ioc_init timed out\n");
3213 	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
3215 		ioc_err(mrioc,
3217 		    (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
3218 		    mrioc->init_cmds.ioc_loginfo);
3223 	mrioc->reply_free_queue_host_index = mrioc->num_reply_bufs;
3224 	writel(mrioc->reply_free_queue_host_index,
3225 	    &mrioc->sysif_regs->reply_free_host_index);
3227 	mrioc->sbq_host_index = mrioc->num_sense_bufs;
3228 	writel(mrioc->sbq_host_index,
3229 	    &mrioc->sysif_regs->sense_buffer_free_host_index);
3231 	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
3232 	mutex_unlock(&mrioc->init_cmds.mutex);
3236 		dma_free_coherent(&mrioc->pdev->dev, data_len, drv_info,
3244  * @mrioc: Adapter instance reference
3252 static void mpi3mr_unmask_events(struct mpi3mr_ioc *mrioc, u16 event)
3263 	mrioc->event_masks[word] &= ~desired_event;
3268  * @mrioc: Adapter instance reference
3275 static int mpi3mr_issue_event_notification(struct mpi3mr_ioc *mrioc)
3282 	mutex_lock(&mrioc->init_cmds.mutex);
3283 	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
3285 		ioc_err(mrioc, "Issue EvtNotify: Init command is in use\n");
3286 		mutex_unlock(&mrioc->init_cmds.mutex);
3289 	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
3290 	mrioc->init_cmds.is_waiting = 1;
3291 	mrioc->init_cmds.callback = NULL;
3296 		    cpu_to_le32(mrioc->event_masks[i]);
3297 	init_completion(&mrioc->init_cmds.done);
3298 	retval = mpi3mr_admin_request_post(mrioc, &evtnotify_req,
3301 		ioc_err(mrioc, "Issue EvtNotify: Admin Post failed\n");
3304 	wait_for_completion_timeout(&mrioc->init_cmds.done,
3306 	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
3307 		ioc_err(mrioc, "event notification timed out\n");
3308 		mpi3mr_check_rh_fault_ioc(mrioc,
3313 	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
3315 		ioc_err(mrioc,
3317 		    (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
3318 		    mrioc->init_cmds.ioc_loginfo);
3324 	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
3325 	mutex_unlock(&mrioc->init_cmds.mutex);
3332  * @mrioc: Adapter instance reference
3341 int mpi3mr_process_event_ack(struct mpi3mr_ioc *mrioc, u8 event,
3348 	mutex_lock(&mrioc->init_cmds.mutex);
3349 	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
3351 		ioc_err(mrioc, "Send EvtAck: Init command is in use\n");
3352 		mutex_unlock(&mrioc->init_cmds.mutex);
3355 	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
3356 	mrioc->init_cmds.is_waiting = 1;
3357 	mrioc->init_cmds.callback = NULL;
3363 	init_completion(&mrioc->init_cmds.done);
3364 	retval = mpi3mr_admin_request_post(mrioc, &evtack_req,
3367 		ioc_err(mrioc, "Send EvtAck: Admin Post failed\n");
3370 	wait_for_completion_timeout(&mrioc->init_cmds.done,
3372 	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
3373 		ioc_err(mrioc, "Issue EvtNotify: command timed out\n");
3374 		if (!(mrioc->init_cmds.state & MPI3MR_CMD_RESET))
3375 			mpi3mr_check_rh_fault_ioc(mrioc,
3380 	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
3382 		ioc_err(mrioc,
3384 		    (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
3385 		    mrioc->init_cmds.ioc_loginfo);
3391 	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
3392 	mutex_unlock(&mrioc->init_cmds.mutex);
3399  * @mrioc: Adapter instance reference
3407 static int mpi3mr_alloc_chain_bufs(struct mpi3mr_ioc *mrioc)
3413 	if (mrioc->chain_sgl_list)
3416 	num_chains = mrioc->max_host_ios / MPI3MR_CHAINBUF_FACTOR;
3424 	mrioc->chain_buf_count = num_chains;
3426 	mrioc->chain_sgl_list = kzalloc(sz, GFP_KERNEL);
3427 	if (!mrioc->chain_sgl_list)
3430 	if (mrioc->max_sgl_entries > (mrioc->facts.max_data_length /
3432 		mrioc->max_sgl_entries = mrioc->facts.max_data_length /
3434 	sz = mrioc->max_sgl_entries * sizeof(struct mpi3_sge_common);
3435 	ioc_info(mrioc, "number of sgl entries=%d chain buffer size=%dKB\n",
3436 			mrioc->max_sgl_entries, sz/1024);
3438 	mrioc->chain_buf_pool = dma_pool_create("chain_buf pool",
3439 	    &mrioc->pdev->dev, sz, 16, 0);
3440 	if (!mrioc->chain_buf_pool) {
3441 		ioc_err(mrioc, "chain buf pool: dma_pool_create failed\n");
3446 		mrioc->chain_sgl_list[i].addr =
3447 		    dma_pool_zalloc(mrioc->chain_buf_pool, GFP_KERNEL,
3448 		    &mrioc->chain_sgl_list[i].dma_addr);
3450 		if (!mrioc->chain_sgl_list[i].addr)
3453 	mrioc->chain_bitmap = bitmap_zalloc(num_chains, GFP_KERNEL);
3454 	if (!mrioc->chain_bitmap)
3464  * @mrioc: Adapter instance reference
3472 static void mpi3mr_port_enable_complete(struct mpi3mr_ioc *mrioc,
3476 	mrioc->scan_started = 0;
3478 		mrioc->scan_failed = MPI3_IOCSTATUS_INTERNAL_ERROR;
3480 		mrioc->scan_failed = drv_cmd->ioc_status;
3486  * @mrioc: Adapter instance reference
3495 int mpi3mr_issue_port_enable(struct mpi3mr_ioc *mrioc, u8 async)
3502 	mutex_lock(&mrioc->init_cmds.mutex);
3503 	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
3505 		ioc_err(mrioc, "Issue PortEnable: Init command is in use\n");
3506 		mutex_unlock(&mrioc->init_cmds.mutex);
3509 	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
3511 		mrioc->init_cmds.is_waiting = 0;
3512 		mrioc->init_cmds.callback = mpi3mr_port_enable_complete;
3514 		mrioc->init_cmds.is_waiting = 1;
3515 		mrioc->init_cmds.callback = NULL;
3516 		init_completion(&mrioc->init_cmds.done);
3521 	retval = mpi3mr_admin_request_post(mrioc, &pe_req, sizeof(pe_req), 1);
3523 		ioc_err(mrioc, "Issue PortEnable: Admin Post failed\n");
3527 		mutex_unlock(&mrioc->init_cmds.mutex);
3531 	wait_for_completion_timeout(&mrioc->init_cmds.done, (pe_timeout * HZ));
3532 	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
3533 		ioc_err(mrioc, "port enable timed out\n");
3535 		mpi3mr_check_rh_fault_ioc(mrioc, MPI3MR_RESET_FROM_PE_TIMEOUT);
3538 	mpi3mr_port_enable_complete(mrioc, &mrioc->init_cmds);
3541 	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
3542 	mutex_unlock(&mrioc->init_cmds.mutex);
3568  * @mrioc: Adapter instance reference
3576 mpi3mr_print_ioc_info(struct mpi3mr_ioc *mrioc)
3582 	struct mpi3mr_compimg_ver *fwver = &mrioc->facts.fw_ver;
3584 	switch (mrioc->facts.personality) {
3596 	ioc_info(mrioc, "Running in %s Personality", personality);
3598 	ioc_info(mrioc, "FW version(%d.%d.%d.%d.%d.%d)\n",
3603 		if (mrioc->facts.protocol_flags &
3614 		if (mrioc->facts.protocol_flags &
3623 	ioc_info(mrioc, "Protocol=(%s), Capabilities=(%s)\n",
3629  * @mrioc: Adapter instance reference
3635 void mpi3mr_cleanup_resources(struct mpi3mr_ioc *mrioc)
3637 	struct pci_dev *pdev = mrioc->pdev;
3639 	mpi3mr_cleanup_isr(mrioc);
3641 	if (mrioc->sysif_regs) {
3642 		iounmap((void __iomem *)mrioc->sysif_regs);
3643 		mrioc->sysif_regs = NULL;
3647 		if (mrioc->bars)
3648 			pci_release_selected_regions(pdev, mrioc->bars);
3655  * @mrioc: Adapter instance reference
3661 int mpi3mr_setup_resources(struct mpi3mr_ioc *mrioc)
3663 	struct pci_dev *pdev = mrioc->pdev;
3667 	u64 dma_mask = mrioc->dma_mask ? mrioc->dma_mask :
3671 		ioc_err(mrioc, "pci_enable_device_mem: failed\n");
3678 		ioc_err(mrioc, "Unable to find MSI-X Capabilities\n");
3682 	mrioc->bars = pci_select_bars(pdev, IORESOURCE_MEM);
3684 	if (pci_request_selected_regions(pdev, mrioc->bars,
3685 	    mrioc->driver_name)) {
3686 		ioc_err(mrioc, "pci_request_selected_regions: failed\n");
3693 			mrioc->sysif_regs_phys = pci_resource_start(pdev, i);
3695 			mrioc->sysif_regs =
3696 			    ioremap(mrioc->sysif_regs_phys, memap_sz);
3706 			ioc_warn(mrioc, "Setting 64 bit DMA mask failed\n");
3712 			mrioc->dma_mask = 0;
3713 			ioc_err(mrioc, "Setting 32 bit DMA mask also failed\n");
3717 	mrioc->dma_mask = dma_mask;
3719 	if (!mrioc->sysif_regs) {
3720 		ioc_err(mrioc,
3727 	mrioc->msix_count = (message_control & 0x3FF) + 1;
3731 	pci_set_drvdata(pdev, mrioc->shost);
3733 	mpi3mr_ioc_disable_intr(mrioc);
3735 	ioc_info(mrioc, "iomem(0x%016llx), mapped(0x%p), size(%d)\n",
3736 	    (unsigned long long)mrioc->sysif_regs_phys,
3737 	    mrioc->sysif_regs, memap_sz);
3738 	ioc_info(mrioc, "Number of MSI-X vectors found in capabilities: (%d)\n",
3739 	    mrioc->msix_count);
3742 		mrioc->requested_poll_qcount = min_t(int, poll_queues,
3743 				mrioc->msix_count - 2);
3747 	mpi3mr_cleanup_resources(mrioc);
3753  * @mrioc: Adapter instance reference
3761 static int mpi3mr_enable_events(struct mpi3mr_ioc *mrioc)
3767 		mrioc->event_masks[i] = -1;
3769 	mpi3mr_unmask_events(mrioc, MPI3_EVENT_DEVICE_ADDED);
3770 	mpi3mr_unmask_events(mrioc, MPI3_EVENT_DEVICE_INFO_CHANGED);
3771 	mpi3mr_unmask_events(mrioc, MPI3_EVENT_DEVICE_STATUS_CHANGE);
3772 	mpi3mr_unmask_events(mrioc, MPI3_EVENT_ENCL_DEVICE_STATUS_CHANGE);
3773 	mpi3mr_unmask_events(mrioc, MPI3_EVENT_ENCL_DEVICE_ADDED);
3774 	mpi3mr_unmask_events(mrioc, MPI3_EVENT_SAS_TOPOLOGY_CHANGE_LIST);
3775 	mpi3mr_unmask_events(mrioc, MPI3_EVENT_SAS_DISCOVERY);
3776 	mpi3mr_unmask_events(mrioc, MPI3_EVENT_SAS_DEVICE_DISCOVERY_ERROR);
3777 	mpi3mr_unmask_events(mrioc, MPI3_EVENT_SAS_BROADCAST_PRIMITIVE);
3778 	mpi3mr_unmask_events(mrioc, MPI3_EVENT_PCIE_TOPOLOGY_CHANGE_LIST);
3779 	mpi3mr_unmask_events(mrioc, MPI3_EVENT_PCIE_ENUMERATION);
3780 	mpi3mr_unmask_events(mrioc, MPI3_EVENT_PREPARE_FOR_RESET);
3781 	mpi3mr_unmask_events(mrioc, MPI3_EVENT_CABLE_MGMT);
3782 	mpi3mr_unmask_events(mrioc, MPI3_EVENT_ENERGY_PACK_CHANGE);
3784 	retval = mpi3mr_issue_event_notification(mrioc);
3786 		ioc_err(mrioc, "failed to issue event notification %d\n",
3793  * @mrioc: Adapter instance reference
3806 int mpi3mr_init_ioc(struct mpi3mr_ioc *mrioc)
3814 	retval = mpi3mr_bring_ioc_ready(mrioc);
3816 		ioc_err(mrioc, "Failed to bring ioc ready: error %d\n",
3821 	retval = mpi3mr_setup_isr(mrioc, 1);
3823 		ioc_err(mrioc, "Failed to setup ISR error %d\n",
3828 	retval = mpi3mr_issue_iocfacts(mrioc, &facts_data);
3830 		ioc_err(mrioc, "Failed to Issue IOC Facts %d\n",
3835 	mrioc->max_host_ios = mrioc->facts.max_reqs - MPI3MR_INTERNAL_CMDS_RESVD;
3836 	mrioc->shost->max_sectors = mrioc->facts.max_data_length / 512;
3837 	mrioc->num_io_throttle_group = mrioc->facts.max_io_throttle_group;
3838 	atomic_set(&mrioc->pend_large_data_sz, 0);
3841 		mrioc->max_host_ios = min_t(int, mrioc->max_host_ios,
3844 	if (!(mrioc->facts.ioc_capabilities &
3846 		mrioc->sas_transport_enabled = 1;
3847 		mrioc->scsi_device_channel = 1;
3848 		mrioc->shost->max_channel = 1;
3849 		mrioc->shost->transportt = mpi3mr_transport_template;
3852 	mrioc->reply_sz = mrioc->facts.reply_sz;
3854 	retval = mpi3mr_check_reset_dma_mask(mrioc);
3856 		ioc_err(mrioc, "Resetting dma mask failed %d\n",
3861 	mpi3mr_print_ioc_info(mrioc);
3863 	if (!mrioc->cfg_page) {
3864 		dprint_init(mrioc, "allocating config page buffers\n");
3865 		mrioc->cfg_page_sz = MPI3MR_DEFAULT_CFG_PAGE_SZ;
3866 		mrioc->cfg_page = dma_alloc_coherent(&mrioc->pdev->dev,
3867 		    mrioc->cfg_page_sz, &mrioc->cfg_page_dma, GFP_KERNEL);
3868 		if (!mrioc->cfg_page) {
3874 	if (!mrioc->init_cmds.reply) {
3875 		retval = mpi3mr_alloc_reply_sense_bufs(mrioc);
3877 			ioc_err(mrioc,
3884 	if (!mrioc->chain_sgl_list) {
3885 		retval = mpi3mr_alloc_chain_bufs(mrioc);
3887 			ioc_err(mrioc, "Failed to allocated chain buffers %d\n",
3893 	retval = mpi3mr_issue_iocinit(mrioc);
3895 		ioc_err(mrioc, "Failed to Issue IOC Init %d\n",
3900 	retval = mpi3mr_print_pkg_ver(mrioc);
3902 		ioc_err(mrioc, "failed to get package version\n");
3906 	retval = mpi3mr_setup_isr(mrioc, 0);
3908 		ioc_err(mrioc, "Failed to re-setup ISR, error %d\n",
3913 	retval = mpi3mr_create_op_queues(mrioc);
3915 		ioc_err(mrioc, "Failed to create OpQueues error %d\n",
3920 	if (!mrioc->pel_seqnum_virt) {
3921 		dprint_init(mrioc, "allocating memory for pel_seqnum_virt\n");
3922 		mrioc->pel_seqnum_sz = sizeof(struct mpi3_pel_seq);
3923 		mrioc->pel_seqnum_virt = dma_alloc_coherent(&mrioc->pdev->dev,
3924 		    mrioc->pel_seqnum_sz, &mrioc->pel_seqnum_dma,
3926 		if (!mrioc->pel_seqnum_virt) {
3932 	if (!mrioc->throttle_groups && mrioc->num_io_throttle_group) {
3933 		dprint_init(mrioc, "allocating memory for throttle groups\n");
3935 		mrioc->throttle_groups = kcalloc(mrioc->num_io_throttle_group, sz, GFP_KERNEL);
3936 		if (!mrioc->throttle_groups) {
3942 	retval = mpi3mr_enable_events(mrioc);
3944 		ioc_err(mrioc, "failed to enable events %d\n",
3949 	ioc_info(mrioc, "controller initialization completed successfully\n");
3954 		ioc_warn(mrioc, "retrying controller initialization, retry_count:%d\n",
3956 		mpi3mr_memset_buffers(mrioc);
3961 	ioc_err(mrioc, "controller initialization failed\n");
3962 	mpi3mr_issue_reset(mrioc, MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,
3964 	mrioc->unrecoverable = 1;
3970  * @mrioc: Adapter instance reference
3982 int mpi3mr_reinit_ioc(struct mpi3mr_ioc *mrioc, u8 is_resume)
3993 	dprint_reset(mrioc, "bringing up the controller to ready state\n");
3994 	retval = mpi3mr_bring_ioc_ready(mrioc);
3996 		ioc_err(mrioc, "failed to bring to ready state\n");
4001 		dprint_reset(mrioc, "setting up single ISR\n");
4002 		retval = mpi3mr_setup_isr(mrioc, 1);
4004 			ioc_err(mrioc, "failed to setup ISR\n");
4008 		mpi3mr_ioc_enable_intr(mrioc);
4010 	dprint_reset(mrioc, "getting ioc_facts\n");
4011 	retval = mpi3mr_issue_iocfacts(mrioc, &facts_data);
4013 		ioc_err(mrioc, "failed to get ioc_facts\n");
4017 	dprint_reset(mrioc, "validating ioc_facts\n");
4018 	retval = mpi3mr_revalidate_factsdata(mrioc);
4020 		ioc_err(mrioc, "failed to revalidate ioc_facts data\n");
4024 	mpi3mr_print_ioc_info(mrioc);
4026 	dprint_reset(mrioc, "sending ioc_init\n");
4027 	retval = mpi3mr_issue_iocinit(mrioc);
4029 		ioc_err(mrioc, "failed to send ioc_init\n");
4033 	dprint_reset(mrioc, "getting package version\n");
4034 	retval = mpi3mr_print_pkg_ver(mrioc);
4036 		ioc_err(mrioc, "failed to get package version\n");
4041 		dprint_reset(mrioc, "setting up multiple ISR\n");
4042 		retval = mpi3mr_setup_isr(mrioc, 0);
4044 			ioc_err(mrioc, "failed to re-setup ISR\n");
4049 	dprint_reset(mrioc, "creating operational queue pairs\n");
4050 	retval = mpi3mr_create_op_queues(mrioc);
4052 		ioc_err(mrioc, "failed to create operational queue pairs\n");
4056 	if (!mrioc->pel_seqnum_virt) {
4057 		dprint_reset(mrioc, "allocating memory for pel_seqnum_virt\n");
4058 		mrioc->pel_seqnum_sz = sizeof(struct mpi3_pel_seq);
4059 		mrioc->pel_seqnum_virt = dma_alloc_coherent(&mrioc->pdev->dev,
4060 		    mrioc->pel_seqnum_sz, &mrioc->pel_seqnum_dma,
4062 		if (!mrioc->pel_seqnum_virt) {
4068 	if (mrioc->shost->nr_hw_queues > mrioc->num_op_reply_q) {
4069 		ioc_err(mrioc,
4071 		    mrioc->shost->nr_hw_queues, mrioc->num_op_reply_q);
4076 	dprint_reset(mrioc, "enabling events\n");
4077 	retval = mpi3mr_enable_events(mrioc);
4079 		ioc_err(mrioc, "failed to enable events\n");
4083 	mrioc->device_refresh_on = 1;
4084 	mpi3mr_add_event_wait_for_device_refresh(mrioc);
4086 	ioc_info(mrioc, "sending port enable\n");
4087 	retval = mpi3mr_issue_port_enable(mrioc, 1);
4089 		ioc_err(mrioc, "failed to issue port enable\n");
4094 		if (mrioc->init_cmds.state == MPI3MR_CMD_NOTUSED)
4096 		if (!pci_device_is_present(mrioc->pdev))
4097 			mrioc->unrecoverable = 1;
4098 		if (mrioc->unrecoverable) {
4102 		ioc_status = readl(&mrioc->sysif_regs->ioc_status);
4105 			mpi3mr_print_fault_info(mrioc);
4106 			mrioc->init_cmds.is_waiting = 0;
4107 			mrioc->init_cmds.callback = NULL;
4108 			mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
4114 		ioc_err(mrioc, "port enable timed out\n");
4115 		mpi3mr_check_rh_fault_ioc(mrioc,
4117 		mrioc->init_cmds.is_waiting = 0;
4118 		mrioc->init_cmds.callback = NULL;
4119 		mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
4121 	} else if (mrioc->scan_failed) {
4122 		ioc_err(mrioc,
4124 		    mrioc->scan_failed);
4126 		ioc_info(mrioc, "port enable completed successfully\n");
4128 	ioc_info(mrioc, "controller %s completed successfully\n",
4134 		ioc_warn(mrioc, "retrying controller %s, retry_count:%d\n",
4136 		mpi3mr_memset_buffers(mrioc);
4141 	ioc_err(mrioc, "controller %s is failed\n",
4143 	mpi3mr_issue_reset(mrioc, MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,
4145 	mrioc->unrecoverable = 1;
4152  * @mrioc: Adapter instance reference
4157 static void mpi3mr_memset_op_reply_q_buffers(struct mpi3mr_ioc *mrioc, u16 qidx)
4159 	struct op_reply_qinfo *op_reply_q = mrioc->op_reply_qinfo + qidx;
4166 	size = op_reply_q->segment_qd * mrioc->op_reply_desc_sz;
4175  * @mrioc: Adapter instance reference
4180 static void mpi3mr_memset_op_req_q_buffers(struct mpi3mr_ioc *mrioc, u16 qidx)
4182 	struct op_req_qinfo *op_req_q = mrioc->req_qinfo + qidx;
4189 	size = op_req_q->segment_qd * mrioc->facts.op_req_sz;
4197  * @mrioc: Adapter instance reference
4205 void mpi3mr_memset_buffers(struct mpi3mr_ioc *mrioc)
4210 	mrioc->change_count = 0;
4211 	mrioc->active_poll_qcount = 0;
4212 	mrioc->default_qcount = 0;
4213 	if (mrioc->admin_req_base)
4214 		memset(mrioc->admin_req_base, 0, mrioc->admin_req_q_sz);
4215 	if (mrioc->admin_reply_base)
4216 		memset(mrioc->admin_reply_base, 0, mrioc->admin_reply_q_sz);
4217 	atomic_set(&mrioc->admin_reply_q_in_use, 0);
4219 	if (mrioc->init_cmds.reply) {
4220 		memset(mrioc->init_cmds.reply, 0, sizeof(*mrioc->init_cmds.reply));
4221 		memset(mrioc->bsg_cmds.reply, 0,
4222 		    sizeof(*mrioc->bsg_cmds.reply));
4223 		memset(mrioc->host_tm_cmds.reply, 0,
4224 		    sizeof(*mrioc->host_tm_cmds.reply));
4225 		memset(mrioc->pel_cmds.reply, 0,
4226 		    sizeof(*mrioc->pel_cmds.reply));
4227 		memset(mrioc->pel_abort_cmd.reply, 0,
4228 		    sizeof(*mrioc->pel_abort_cmd.reply));
4229 		memset(mrioc->transport_cmds.reply, 0,
4230 		    sizeof(*mrioc->transport_cmds.reply));
4232 			memset(mrioc->dev_rmhs_cmds[i].reply, 0,
4233 			    sizeof(*mrioc->dev_rmhs_cmds[i].reply));
4235 			memset(mrioc->evtack_cmds[i].reply, 0,
4236 			    sizeof(*mrioc->evtack_cmds[i].reply));
4237 		bitmap_clear(mrioc->removepend_bitmap, 0,
4238 			     mrioc->dev_handle_bitmap_bits);
4239 		bitmap_clear(mrioc->devrem_bitmap, 0, MPI3MR_NUM_DEVRMCMD);
4240 		bitmap_clear(mrioc->evtack_cmds_bitmap, 0,
4244 	for (i = 0; i < mrioc->num_queues; i++) {
4245 		mrioc->op_reply_qinfo[i].qid = 0;
4246 		mrioc->op_reply_qinfo[i].ci = 0;
4247 		mrioc->op_reply_qinfo[i].num_replies = 0;
4248 		mrioc->op_reply_qinfo[i].ephase = 0;
4249 		atomic_set(&mrioc->op_reply_qinfo[i].pend_ios, 0);
4250 		atomic_set(&mrioc->op_reply_qinfo[i].in_use, 0);
4251 		mpi3mr_memset_op_reply_q_buffers(mrioc, i);
4253 		mrioc->req_qinfo[i].ci = 0;
4254 		mrioc->req_qinfo[i].pi = 0;
4255 		mrioc->req_qinfo[i].num_requests = 0;
4256 		mrioc->req_qinfo[i].qid = 0;
4257 		mrioc->req_qinfo[i].reply_qid = 0;
4258 		spin_lock_init(&mrioc->req_qinfo[i].q_lock);
4259 		mpi3mr_memset_op_req_q_buffers(mrioc, i);
4262 	atomic_set(&mrioc->pend_large_data_sz, 0);
4263 	if (mrioc->throttle_groups) {
4264 		tg = mrioc->throttle_groups;
4265 		for (i = 0; i < mrioc->num_io_throttle_group; i++, tg++) {
4281  * @mrioc: Adapter instance reference
4287 void mpi3mr_free_mem(struct mpi3mr_ioc *mrioc)
4292 	mpi3mr_free_enclosure_list(mrioc);
4294 	if (mrioc->sense_buf_pool) {
4295 		if (mrioc->sense_buf)
4296 			dma_pool_free(mrioc->sense_buf_pool, mrioc->sense_buf,
4297 			    mrioc->sense_buf_dma);
4298 		dma_pool_destroy(mrioc->sense_buf_pool);
4299 		mrioc->sense_buf = NULL;
4300 		mrioc->sense_buf_pool = NULL;
4302 	if (mrioc->sense_buf_q_pool) {
4303 		if (mrioc->sense_buf_q)
4304 			dma_pool_free(mrioc->sense_buf_q_pool,
4305 			    mrioc->sense_buf_q, mrioc->sense_buf_q_dma);
4306 		dma_pool_destroy(mrioc->sense_buf_q_pool);
4307 		mrioc->sense_buf_q = NULL;
4308 		mrioc->sense_buf_q_pool = NULL;
4311 	if (mrioc->reply_buf_pool) {
4312 		if (mrioc->reply_buf)
4313 			dma_pool_free(mrioc->reply_buf_pool, mrioc->reply_buf,
4314 			    mrioc->reply_buf_dma);
4315 		dma_pool_destroy(mrioc->reply_buf_pool);
4316 		mrioc->reply_buf = NULL;
4317 		mrioc->reply_buf_pool = NULL;
4319 	if (mrioc->reply_free_q_pool) {
4320 		if (mrioc->reply_free_q)
4321 			dma_pool_free(mrioc->reply_free_q_pool,
4322 			    mrioc->reply_free_q, mrioc->reply_free_q_dma);
4323 		dma_pool_destroy(mrioc->reply_free_q_pool);
4324 		mrioc->reply_free_q = NULL;
4325 		mrioc->reply_free_q_pool = NULL;
4328 	for (i = 0; i < mrioc->num_op_req_q; i++)
4329 		mpi3mr_free_op_req_q_segments(mrioc, i);
4331 	for (i = 0; i < mrioc->num_op_reply_q; i++)
4332 		mpi3mr_free_op_reply_q_segments(mrioc, i);
4334 	for (i = 0; i < mrioc->intr_info_count; i++) {
4335 		intr_info = mrioc->intr_info + i;
4339 	kfree(mrioc->req_qinfo);
4340 	mrioc->req_qinfo = NULL;
4341 	mrioc->num_op_req_q = 0;
4343 	kfree(mrioc->op_reply_qinfo);
4344 	mrioc->op_reply_qinfo = NULL;
4345 	mrioc->num_op_reply_q = 0;
4347 	kfree(mrioc->init_cmds.reply);
4348 	mrioc->init_cmds.reply = NULL;
4350 	kfree(mrioc->bsg_cmds.reply);
4351 	mrioc->bsg_cmds.reply = NULL;
4353 	kfree(mrioc->host_tm_cmds.reply);
4354 	mrioc->host_tm_cmds.reply = NULL;
4356 	kfree(mrioc->pel_cmds.reply);
4357 	mrioc->pel_cmds.reply = NULL;
4359 	kfree(mrioc->pel_abort_cmd.reply);
4360 	mrioc->pel_abort_cmd.reply = NULL;
4363 		kfree(mrioc->evtack_cmds[i].reply);
4364 		mrioc->evtack_cmds[i].reply = NULL;
4367 	bitmap_free(mrioc->removepend_bitmap);
4368 	mrioc->removepend_bitmap = NULL;
4370 	bitmap_free(mrioc->devrem_bitmap);
4371 	mrioc->devrem_bitmap = NULL;
4373 	bitmap_free(mrioc->evtack_cmds_bitmap);
4374 	mrioc->evtack_cmds_bitmap = NULL;
4376 	bitmap_free(mrioc->chain_bitmap);
4377 	mrioc->chain_bitmap = NULL;
4379 	kfree(mrioc->transport_cmds.reply);
4380 	mrioc->transport_cmds.reply = NULL;
4383 		kfree(mrioc->dev_rmhs_cmds[i].reply);
4384 		mrioc->dev_rmhs_cmds[i].reply = NULL;
4387 	if (mrioc->chain_buf_pool) {
4388 		for (i = 0; i < mrioc->chain_buf_count; i++) {
4389 			if (mrioc->chain_sgl_list[i].addr) {
4390 				dma_pool_free(mrioc->chain_buf_pool,
4391 				    mrioc->chain_sgl_list[i].addr,
4392 				    mrioc->chain_sgl_list[i].dma_addr);
4393 				mrioc->chain_sgl_list[i].addr = NULL;
4396 		dma_pool_destroy(mrioc->chain_buf_pool);
4397 		mrioc->chain_buf_pool = NULL;
4400 	kfree(mrioc->chain_sgl_list);
4401 	mrioc->chain_sgl_list = NULL;
4403 	if (mrioc->admin_reply_base) {
4404 		dma_free_coherent(&mrioc->pdev->dev, mrioc->admin_reply_q_sz,
4405 		    mrioc->admin_reply_base, mrioc->admin_reply_dma);
4406 		mrioc->admin_reply_base = NULL;
4408 	if (mrioc->admin_req_base) {
4409 		dma_free_coherent(&mrioc->pdev->dev, mrioc->admin_req_q_sz,
4410 		    mrioc->admin_req_base, mrioc->admin_req_dma);
4411 		mrioc->admin_req_base = NULL;
4413 	if (mrioc->cfg_page) {
4414 		dma_free_coherent(&mrioc->pdev->dev, mrioc->cfg_page_sz,
4415 		    mrioc->cfg_page, mrioc->cfg_page_dma);
4416 		mrioc->cfg_page = NULL;
4418 	if (mrioc->pel_seqnum_virt) {
4419 		dma_free_coherent(&mrioc->pdev->dev, mrioc->pel_seqnum_sz,
4420 		    mrioc->pel_seqnum_virt, mrioc->pel_seqnum_dma);
4421 		mrioc->pel_seqnum_virt = NULL;
4424 	kfree(mrioc->throttle_groups);
4425 	mrioc->throttle_groups = NULL;
4427 	kfree(mrioc->logdata_buf);
4428 	mrioc->logdata_buf = NULL;
4434  * @mrioc: Adapter instance reference
4441 static void mpi3mr_issue_ioc_shutdown(struct mpi3mr_ioc *mrioc)
4447 	ioc_info(mrioc, "Issuing shutdown Notification\n");
4448 	if (mrioc->unrecoverable) {
4449 		ioc_warn(mrioc,
4453 	ioc_status = readl(&mrioc->sysif_regs->ioc_status);
4456 		ioc_info(mrioc, "shutdown already in progress\n");
4460 	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
4464 	writel(ioc_config, &mrioc->sysif_regs->ioc_configuration);
4466 	if (mrioc->facts.shutdown_timeout)
4467 		timeout = mrioc->facts.shutdown_timeout * 10;
4470 		ioc_status = readl(&mrioc->sysif_regs->ioc_status);
4479 	ioc_status = readl(&mrioc->sysif_regs->ioc_status);
4480 	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
4485 			ioc_warn(mrioc,
4489 	ioc_info(mrioc,
4497  * @mrioc: Adapter instance reference
4504 void mpi3mr_cleanup_ioc(struct mpi3mr_ioc *mrioc)
4508 	dprint_exit(mrioc, "cleaning up the controller\n");
4509 	mpi3mr_ioc_disable_intr(mrioc);
4511 	ioc_state = mpi3mr_get_iocstate(mrioc);
4513 	if ((!mrioc->unrecoverable) && (!mrioc->reset_in_progress) &&
4515 		if (mpi3mr_issue_and_process_mur(mrioc,
4517 			mpi3mr_issue_reset(mrioc,
4520 		mpi3mr_issue_ioc_shutdown(mrioc);
4522 	dprint_exit(mrioc, "controller cleanup completed\n");
4527  * @mrioc: Adapter instance reference
4535 static inline void mpi3mr_drv_cmd_comp_reset(struct mpi3mr_ioc *mrioc,
4545 			cmdptr->callback(mrioc, cmdptr);
4551  * @mrioc: Adapter instance reference
4557 void mpi3mr_flush_drv_cmds(struct mpi3mr_ioc *mrioc)
4562 	cmdptr = &mrioc->init_cmds;
4563 	mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
4565 	cmdptr = &mrioc->cfg_cmds;
4566 	mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
4568 	cmdptr = &mrioc->bsg_cmds;
4569 	mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
4570 	cmdptr = &mrioc->host_tm_cmds;
4571 	mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
4574 		cmdptr = &mrioc->dev_rmhs_cmds[i];
4575 		mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
4579 		cmdptr = &mrioc->evtack_cmds[i];
4580 		mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
4583 	cmdptr = &mrioc->pel_cmds;
4584 	mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
4586 	cmdptr = &mrioc->pel_abort_cmd;
4587 	mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
4589 	cmdptr = &mrioc->transport_cmds;
4590 	mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
4595  * @mrioc: Adapter instance reference
4602 static void mpi3mr_pel_wait_post(struct mpi3mr_ioc *mrioc,
4607 	mrioc->pel_abort_requested = false;
4618 	pel_wait.starting_sequence_number = cpu_to_le32(mrioc->pel_newest_seqnum);
4619 	pel_wait.locale = cpu_to_le16(mrioc->pel_locale);
4620 	pel_wait.class = cpu_to_le16(mrioc->pel_class);
4622 	dprint_bsg_info(mrioc, "sending pel_wait seqnum(%d), class(%d), locale(0x%08x)\n",
4623 	    mrioc->pel_newest_seqnum, mrioc->pel_class, mrioc->pel_locale);
4625 	if (mpi3mr_admin_request_post(mrioc, &pel_wait, sizeof(pel_wait), 0)) {
4626 		dprint_bsg_err(mrioc,
4631 		mrioc->pel_enabled = false;
4637  * @mrioc: Adapter instance reference
4645 int mpi3mr_pel_get_seqnum_post(struct mpi3mr_ioc *mrioc,
4653 	mrioc->pel_cmds.state = MPI3MR_CMD_PENDING;
4654 	mrioc->pel_cmds.is_waiting = 0;
4655 	mrioc->pel_cmds.ioc_status = 0;
4656 	mrioc->pel_cmds.ioc_loginfo = 0;
4657 	mrioc->pel_cmds.callback = mpi3mr_pel_get_seqnum_complete;
4662 	    mrioc->pel_seqnum_sz, mrioc->pel_seqnum_dma);
4664 	retval = mpi3mr_admin_request_post(mrioc, &pel_getseq_req,
4672 		mrioc->pel_enabled = false;
4680  * @mrioc: Adapter instance reference
4691 static void mpi3mr_pel_wait_complete(struct mpi3mr_ioc *mrioc,
4703 		ioc_err(mrioc, "%s: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
4705 		dprint_bsg_err(mrioc,
4715 		dprint_bsg_err(mrioc,
4723 		ioc_err(mrioc, "%s: Failed pe_log_status(0x%04x)\n",
4725 		dprint_bsg_err(mrioc,
4734 			dprint_bsg_err(mrioc, "pel_wait: retrying(%d)\n",
4736 			mpi3mr_pel_wait_post(mrioc, drv_cmd);
4739 		dprint_bsg_err(mrioc,
4745 	if (!mrioc->pel_abort_requested) {
4746 		mrioc->pel_cmds.retry_count = 0;
4747 		mpi3mr_pel_get_seqnum_post(mrioc, &mrioc->pel_cmds);
4752 	mrioc->pel_enabled = false;
4761  * @mrioc: Adapter instance reference
4770 void mpi3mr_pel_get_seqnum_complete(struct mpi3mr_ioc *mrioc,
4778 	pel_seqnum_virt = (struct mpi3_pel_seq *)mrioc->pel_seqnum_virt;
4785 		dprint_bsg_err(mrioc,
4794 		dprint_bsg_err(mrioc,
4800 		dprint_bsg_err(mrioc,
4809 			dprint_bsg_err(mrioc,
4812 			mpi3mr_pel_get_seqnum_post(mrioc, drv_cmd);
4816 		dprint_bsg_err(mrioc,
4821 	mrioc->pel_newest_seqnum = le32_to_cpu(pel_seqnum_virt->newest) + 1;
4823 	mpi3mr_pel_wait_post(mrioc, drv_cmd);
4827 	mrioc->pel_enabled = false;
4836  * @mrioc: Adapter instance reference
4855 int mpi3mr_soft_reset_handler(struct mpi3mr_ioc *mrioc,
4863 	dprint_reset(mrioc,
4865 	    mrioc->diagsave_timeout);
4866 	while (mrioc->diagsave_timeout)
4872 	dprint_reset(mrioc, "soft_reset_handler: acquiring reset_mutex\n");
4873 	if (!mutex_trylock(&mrioc->reset_mutex)) {
4874 		ioc_info(mrioc,
4879 		} while (mrioc->reset_in_progress == 1);
4880 		ioc_info(mrioc,
4882 		    mrioc->prev_reset_result,
4884 		return mrioc->prev_reset_result;
4886 	ioc_info(mrioc, "controller reset is triggered by %s\n",
4889 	mrioc->device_refresh_on = 0;
4890 	mrioc->reset_in_progress = 1;
4891 	mrioc->stop_bsgs = 1;
4892 	mrioc->prev_reset_result = -1;
4898 			mrioc->event_masks[i] = -1;
4900 		dprint_reset(mrioc, "soft_reset_handler: masking events\n");
4901 		mpi3mr_issue_event_notification(mrioc);
4904 	mpi3mr_wait_for_host_io(mrioc, MPI3MR_RESET_HOST_IOWAIT_TIMEOUT);
4906 	mpi3mr_ioc_disable_intr(mrioc);
4909 		mpi3mr_set_diagsave(mrioc);
4910 		retval = mpi3mr_issue_reset(mrioc,
4915 				    readl(&mrioc->sysif_regs->host_diagnostic);
4924 	retval = mpi3mr_issue_reset(mrioc,
4927 		ioc_err(mrioc, "Failed to issue soft reset to the ioc\n");
4930 	if (mrioc->num_io_throttle_group !=
4931 	    mrioc->facts.max_io_throttle_group) {
4932 		ioc_err(mrioc,
4934 		    mrioc->num_io_throttle_group,
4935 		    mrioc->facts.max_io_throttle_group);
4940 	mpi3mr_flush_delayed_cmd_lists(mrioc);
4941 	mpi3mr_flush_drv_cmds(mrioc);
4942 	bitmap_clear(mrioc->devrem_bitmap, 0, MPI3MR_NUM_DEVRMCMD);
4943 	bitmap_clear(mrioc->removepend_bitmap, 0,
4944 		     mrioc->dev_handle_bitmap_bits);
4945 	bitmap_clear(mrioc->evtack_cmds_bitmap, 0, MPI3MR_NUM_EVTACKCMD);
4946 	mpi3mr_flush_host_io(mrioc);
4947 	mpi3mr_cleanup_fwevt_list(mrioc);
4948 	mpi3mr_invalidate_devhandles(mrioc);
4949 	mpi3mr_free_enclosure_list(mrioc);
4951 	if (mrioc->prepare_for_reset) {
4952 		mrioc->prepare_for_reset = 0;
4953 		mrioc->prepare_for_reset_timeout_counter = 0;
4955 	mpi3mr_memset_buffers(mrioc);
4956 	retval = mpi3mr_reinit_ioc(mrioc, 0);
4959 		    mrioc->name, reset_reason);
4966 		mrioc->diagsave_timeout = 0;
4967 		mrioc->reset_in_progress = 0;
4968 		mrioc->pel_abort_requested = 0;
4969 		if (mrioc->pel_enabled) {
4970 			mrioc->pel_cmds.retry_count = 0;
4971 			mpi3mr_pel_wait_post(mrioc, &mrioc->pel_cmds);
4974 		mrioc->device_refresh_on = 0;
4976 		mrioc->ts_update_counter = 0;
4977 		spin_lock_irqsave(&mrioc->watchdog_lock, flags);
4978 		if (mrioc->watchdog_work_q)
4979 			queue_delayed_work(mrioc->watchdog_work_q,
4980 			    &mrioc->watchdog_work,
4982 		spin_unlock_irqrestore(&mrioc->watchdog_lock, flags);
4983 		mrioc->stop_bsgs = 0;
4984 		if (mrioc->pel_enabled)
4987 		mpi3mr_issue_reset(mrioc,
4989 		mrioc->device_refresh_on = 0;
4990 		mrioc->unrecoverable = 1;
4991 		mrioc->reset_in_progress = 0;
4993 		mpi3mr_flush_cmds_for_unrecovered_controller(mrioc);
4995 	mrioc->prev_reset_result = retval;
4996 	mutex_unlock(&mrioc->reset_mutex);
4997 	ioc_info(mrioc, "controller reset is %s\n",
5005  * @mrioc: Adapter instance reference
5014 static void mpi3mr_free_config_dma_memory(struct mpi3mr_ioc *mrioc,
5017 	if ((mem_desc->size > mrioc->cfg_page_sz) && mem_desc->addr) {
5018 		dma_free_coherent(&mrioc->pdev->dev, mem_desc->size,
5026  * @mrioc: Adapter instance reference
5035 static int mpi3mr_alloc_config_dma_memory(struct mpi3mr_ioc *mrioc,
5038 	if (mem_desc->size > mrioc->cfg_page_sz) {
5039 		mem_desc->addr = dma_alloc_coherent(&mrioc->pdev->dev,
5044 		mem_desc->addr = mrioc->cfg_page;
5045 		mem_desc->dma_addr = mrioc->cfg_page_dma;
5046 		memset(mem_desc->addr, 0, mrioc->cfg_page_sz);
5053  * @mrioc: Adapter instance reference
5068 static int mpi3mr_post_cfg_req(struct mpi3mr_ioc *mrioc,
5073 	mutex_lock(&mrioc->cfg_cmds.mutex);
5074 	if (mrioc->cfg_cmds.state & MPI3MR_CMD_PENDING) {
5076 		ioc_err(mrioc, "sending config request failed due to command in use\n");
5077 		mutex_unlock(&mrioc->cfg_cmds.mutex);
5080 	mrioc->cfg_cmds.state = MPI3MR_CMD_PENDING;
5081 	mrioc->cfg_cmds.is_waiting = 1;
5082 	mrioc->cfg_cmds.callback = NULL;
5083 	mrioc->cfg_cmds.ioc_status = 0;
5084 	mrioc->cfg_cmds.ioc_loginfo = 0;
5089 	init_completion(&mrioc->cfg_cmds.done);
5090 	dprint_cfg_info(mrioc, "posting config request\n");
5091 	if (mrioc->logging_level & MPI3_DEBUG_CFG_INFO)
5094 	retval = mpi3mr_admin_request_post(mrioc, cfg_req, sizeof(*cfg_req), 1);
5096 		ioc_err(mrioc, "posting config request failed\n");
5099 	wait_for_completion_timeout(&mrioc->cfg_cmds.done, (timeout * HZ));
5100 	if (!(mrioc->cfg_cmds.state & MPI3MR_CMD_COMPLETE)) {
5101 		mpi3mr_check_rh_fault_ioc(mrioc,
5103 		ioc_err(mrioc, "config request timed out\n");
5107 	*ioc_status = mrioc->cfg_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK;
5109 		dprint_cfg_err(mrioc,
5111 		    *ioc_status, mrioc->cfg_cmds.ioc_loginfo);
5114 	mrioc->cfg_cmds.state = MPI3MR_CMD_NOTUSED;
5115 	mutex_unlock(&mrioc->cfg_cmds.mutex);
5123  * @mrioc: Adapter instance reference
5157 static int mpi3mr_process_cfg_req(struct mpi3mr_ioc *mrioc,
5173 			ioc_err(mrioc, "null config header passed for config action(%d), page_type(0x%02x), page_num(%d)\n",
5196 			ioc_err(mrioc,
5206 	if (mpi3mr_alloc_config_dma_memory(mrioc, &mem_desc))
5216 		dprint_cfg_info(mrioc, "config buffer to be written\n");
5217 		if (mrioc->logging_level & MPI3_DEBUG_CFG_INFO)
5221 	if (mpi3mr_post_cfg_req(mrioc, cfg_req, timeout, ioc_status))
5230 		dprint_cfg_info(mrioc, "config buffer read\n");
5231 		if (mrioc->logging_level & MPI3_DEBUG_CFG_INFO)
5236 	mpi3mr_free_config_dma_memory(mrioc, &mem_desc);
5242  * @mrioc: Adapter instance reference
5257 int mpi3mr_cfg_get_dev_pg0(struct mpi3mr_ioc *mrioc, u16 *ioc_status,
5274 	if (mpi3mr_process_cfg_req(mrioc, &cfg_req, NULL,
5276 		ioc_err(mrioc, "device page0 header read failed\n");
5280 		ioc_err(mrioc, "device page0 header read failed with ioc_status(0x%04x)\n",
5288 	if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr,
5290 		ioc_err(mrioc, "device page0 read failed\n");
5301  * @mrioc: Adapter instance reference
5316 int mpi3mr_cfg_get_sas_phy_pg0(struct mpi3mr_ioc *mrioc, u16 *ioc_status,
5334 	if (mpi3mr_process_cfg_req(mrioc, &cfg_req, NULL,
5336 		ioc_err(mrioc, "sas phy page0 header read failed\n");
5340 		ioc_err(mrioc, "sas phy page0 header read failed with ioc_status(0x%04x)\n",
5348 	if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr,
5350 		ioc_err(mrioc, "sas phy page0 read failed\n");
5360  * @mrioc: Adapter instance reference
5375 int mpi3mr_cfg_get_sas_phy_pg1(struct mpi3mr_ioc *mrioc, u16 *ioc_status,
5393 	if (mpi3mr_process_cfg_req(mrioc, &cfg_req, NULL,
5395 		ioc_err(mrioc, "sas phy page1 header read failed\n");
5399 		ioc_err(mrioc, "sas phy page1 header read failed with ioc_status(0x%04x)\n",
5407 	if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr,
5409 		ioc_err(mrioc, "sas phy page1 read failed\n");
5420  * @mrioc: Adapter instance reference
5435 int mpi3mr_cfg_get_sas_exp_pg0(struct mpi3mr_ioc *mrioc, u16 *ioc_status,
5453 	if (mpi3mr_process_cfg_req(mrioc, &cfg_req, NULL,
5455 		ioc_err(mrioc, "expander page0 header read failed\n");
5459 		ioc_err(mrioc, "expander page0 header read failed with ioc_status(0x%04x)\n",
5468 	if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr,
5470 		ioc_err(mrioc, "expander page0 read failed\n");
5480  * @mrioc: Adapter instance reference
5495 int mpi3mr_cfg_get_sas_exp_pg1(struct mpi3mr_ioc *mrioc, u16 *ioc_status,
5513 	if (mpi3mr_process_cfg_req(mrioc, &cfg_req, NULL,
5515 		ioc_err(mrioc, "expander page1 header read failed\n");
5519 		ioc_err(mrioc, "expander page1 header read failed with ioc_status(0x%04x)\n",
5528 	if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr,
5530 		ioc_err(mrioc, "expander page1 read failed\n");
5540  * @mrioc: Adapter instance reference
5555 int mpi3mr_cfg_get_enclosure_pg0(struct mpi3mr_ioc *mrioc, u16 *ioc_status,
5573 	if (mpi3mr_process_cfg_req(mrioc, &cfg_req, NULL,
5575 		ioc_err(mrioc, "enclosure page0 header read failed\n");
5579 		ioc_err(mrioc, "enclosure page0 header read failed with ioc_status(0x%04x)\n",
5587 	if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr,
5589 		ioc_err(mrioc, "enclosure page0 read failed\n");
5600  * @mrioc: Adapter instance reference
5610 int mpi3mr_cfg_get_sas_io_unit_pg0(struct mpi3mr_ioc *mrioc,
5627 	if (mpi3mr_process_cfg_req(mrioc, &cfg_req, NULL,
5629 		ioc_err(mrioc, "sas io unit page0 header read failed\n");
5633 		ioc_err(mrioc, "sas io unit page0 header read failed with ioc_status(0x%04x)\n",
5639 	if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr,
5641 		ioc_err(mrioc, "sas io unit page0 read failed\n");
5645 		ioc_err(mrioc, "sas io unit page0 read failed with ioc_status(0x%04x)\n",
5656  * @mrioc: Adapter instance reference
5666 int mpi3mr_cfg_get_sas_io_unit_pg1(struct mpi3mr_ioc *mrioc,
5683 	if (mpi3mr_process_cfg_req(mrioc, &cfg_req, NULL,
5685 		ioc_err(mrioc, "sas io unit page1 header read failed\n");
5689 		ioc_err(mrioc, "sas io unit page1 header read failed with ioc_status(0x%04x)\n",
5695 	if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr,
5697 		ioc_err(mrioc, "sas io unit page1 read failed\n");
5701 		ioc_err(mrioc, "sas io unit page1 read failed with ioc_status(0x%04x)\n",
5712  * @mrioc: Adapter instance reference
5723 int mpi3mr_cfg_set_sas_io_unit_pg1(struct mpi3mr_ioc *mrioc,
5739 	if (mpi3mr_process_cfg_req(mrioc, &cfg_req, NULL,
5741 		ioc_err(mrioc, "sas io unit page1 header read failed\n");
5745 		ioc_err(mrioc, "sas io unit page1 header read failed with ioc_status(0x%04x)\n",
5751 	if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr,
5753 		ioc_err(mrioc, "sas io unit page1 write current failed\n");
5757 		ioc_err(mrioc, "sas io unit page1 write current failed with ioc_status(0x%04x)\n",
5764 	if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr,
5766 		ioc_err(mrioc, "sas io unit page1 write persistent failed\n");
5770 		ioc_err(mrioc, "sas io unit page1 write persistent failed with ioc_status(0x%04x)\n",
5781  * @mrioc: Adapter instance reference
5791 int mpi3mr_cfg_get_driver_pg1(struct mpi3mr_ioc *mrioc,
5808 	if (mpi3mr_process_cfg_req(mrioc, &cfg_req, NULL,
5810 		ioc_err(mrioc, "driver page1 header read failed\n");
5814 		ioc_err(mrioc, "driver page1 header read failed with ioc_status(0x%04x)\n",
5820 	if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr,
5822 		ioc_err(mrioc, "driver page1 read failed\n");
5826 		ioc_err(mrioc, "driver page1 read failed with ioc_status(0x%04x)\n",

Indexes created Thu Nov 07 10:32:03 CST 2024