Lines Matching refs:efx
23 #include "efx.h"
194 static int ef4_soft_enable_interrupts(struct ef4_nic *efx);
195 static void ef4_soft_disable_interrupts(struct ef4_nic *efx);
197 static void ef4_remove_channels(struct ef4_nic *efx);
199 static void ef4_remove_port(struct ef4_nic *efx);
201 static void ef4_fini_napi(struct ef4_nic *efx);
203 static void ef4_fini_struct(struct ef4_nic *efx);
204 static void ef4_start_all(struct ef4_nic *efx);
205 static void ef4_stop_all(struct ef4_nic *efx);
207 #define EF4_ASSERT_RESET_SERIALISED(efx) \
209 if ((efx->state == STATE_READY) || \
210 (efx->state == STATE_RECOVERY) || \
211 (efx->state == STATE_DISABLED)) \
215 static int ef4_check_disabled(struct ef4_nic *efx)
217 if (efx->state == STATE_DISABLED || efx->state == STATE_RECOVERY) {
218 netif_err(efx, drv, efx->net_dev,
276 static void ef4_update_irq_mod(struct ef4_nic *efx, struct ef4_channel *channel)
278 int step = efx->irq_mod_step_us;
283 efx->type->push_irq_moderation(channel);
287 efx->irq_rx_moderation_us) {
289 efx->type->push_irq_moderation(channel);
301 struct ef4_nic *efx = channel->efx;
304 netif_vdbg(efx, intr, efx->net_dev,
312 efx->irq_rx_adaptive &&
314 ef4_update_irq_mod(efx, channel);
338 struct ef4_nic *efx = channel->efx;
341 netif_dbg(efx, probe, efx->net_dev,
346 entries = roundup_pow_of_two(efx->rxq_entries + efx->txq_entries + 128);
356 struct ef4_nic *efx = channel->efx;
361 netif_dbg(efx, drv, efx->net_dev,
366 efx->type->push_irq_moderation(channel);
376 netif_dbg(channel->efx, ifup, channel->efx->net_dev,
402 netif_dbg(channel->efx, drv, channel->efx->net_dev,
411 netif_dbg(channel->efx, drv, channel->efx->net_dev,
425 ef4_alloc_channel(struct ef4_nic *efx, int i, struct ef4_channel *old_channel)
436 channel->efx = efx;
442 tx_queue->efx = efx;
448 rx_queue->efx = efx;
499 netif_dbg(channel->efx, probe, channel->efx->net_dev,
532 struct ef4_nic *efx = channel->efx;
537 if (efx->tx_channel_offset == 0) {
539 } else if (channel->channel < efx->tx_channel_offset) {
543 number -= efx->tx_channel_offset;
545 snprintf(buf, len, "%s%s-%d", efx->name, type, number);
548 static void ef4_set_channel_names(struct ef4_nic *efx)
552 ef4_for_each_channel(channel, efx)
554 efx->msi_context[channel->channel].name,
555 sizeof(efx->msi_context[0].name));
558 static int ef4_probe_channels(struct ef4_nic *efx)
564 efx->next_buffer_table = 0;
571 ef4_for_each_channel_rev(channel, efx) {
574 netif_err(efx, probe, efx->net_dev,
580 ef4_set_channel_names(efx);
585 ef4_remove_channels(efx);
593 static void ef4_start_datapath(struct ef4_nic *efx)
595 netdev_features_t old_features = efx->net_dev->features;
596 bool old_rx_scatter = efx->rx_scatter;
606 efx->rx_dma_len = (efx->rx_prefix_size +
607 EF4_MAX_FRAME_LEN(efx->net_dev->mtu) +
608 efx->type->rx_buffer_padding);
610 efx->rx_ip_align + efx->rx_dma_len);
612 efx->rx_scatter = efx->type->always_rx_scatter;
613 efx->rx_buffer_order = 0;
614 } else if (efx->type->can_rx_scatter) {
620 efx->rx_scatter = true;
621 efx->rx_dma_len = EF4_RX_USR_BUF_SIZE;
622 efx->rx_buffer_order = 0;
624 efx->rx_scatter = false;
625 efx->rx_buffer_order = get_order(rx_buf_len);
628 ef4_rx_config_page_split(efx);
629 if (efx->rx_buffer_order)
630 netif_dbg(efx, drv, efx->net_dev,
632 efx->rx_dma_len, efx->rx_buffer_order,
633 efx->rx_pages_per_batch);
635 netif_dbg(efx, drv, efx->net_dev,
637 efx->rx_dma_len, efx->rx_page_buf_step,
638 efx->rx_bufs_per_page, efx->rx_pages_per_batch);
643 efx->net_dev->hw_features |= efx->net_dev->features;
644 efx->net_dev->hw_features &= ~efx->fixed_features;
645 efx->net_dev->features |= efx->fixed_features;
646 if (efx->net_dev->features != old_features)
647 netdev_features_change(efx->net_dev);
650 if (efx->rx_scatter != old_rx_scatter)
651 efx->type->filter_update_rx_scatter(efx);
660 efx->txq_stop_thresh = efx->txq_entries - ef4_tx_max_skb_descs(efx);
661 efx->txq_wake_thresh = efx->txq_stop_thresh / 2;
664 ef4_for_each_channel(channel, efx) {
667 atomic_inc(&efx->active_queues);
672 atomic_inc(&efx->active_queues);
681 if (netif_device_present(efx->net_dev))
682 netif_tx_wake_all_queues(efx->net_dev);
685 static void ef4_stop_datapath(struct ef4_nic *efx)
692 EF4_ASSERT_RESET_SERIALISED(efx);
693 BUG_ON(efx->port_enabled);
696 ef4_for_each_channel(channel, efx) {
701 ef4_for_each_channel(channel, efx) {
714 rc = efx->type->fini_dmaq(efx);
715 if (rc && EF4_WORKAROUND_7803(efx)) {
721 netif_err(efx, drv, efx->net_dev,
723 ef4_schedule_reset(efx, RESET_TYPE_ALL);
725 netif_err(efx, drv, efx->net_dev, "failed to flush queues\n");
727 netif_dbg(efx, drv, efx->net_dev,
731 ef4_for_each_channel(channel, efx) {
744 netif_dbg(channel->efx, drv, channel->efx->net_dev,
755 static void ef4_remove_channels(struct ef4_nic *efx)
759 ef4_for_each_channel(channel, efx)
764 ef4_realloc_channels(struct ef4_nic *efx, u32 rxq_entries, u32 txq_entries)
771 rc = ef4_check_disabled(efx);
778 ef4_for_each_channel(channel, efx) {
797 ef4_device_detach_sync(efx);
798 ef4_stop_all(efx);
799 ef4_soft_disable_interrupts(efx);
803 for (i = 0; i < efx->n_channels; i++) {
804 channel = efx->channel[i];
815 old_rxq_entries = efx->rxq_entries;
816 old_txq_entries = efx->txq_entries;
817 efx->rxq_entries = rxq_entries;
818 efx->txq_entries = txq_entries;
819 for (i = 0; i < efx->n_channels; i++) {
820 channel = efx->channel[i];
821 efx->channel[i] = other_channel[i];
826 efx->next_buffer_table = next_buffer_table;
828 for (i = 0; i < efx->n_channels; i++) {
829 channel = efx->channel[i];
835 ef4_init_napi_channel(efx->channel[i]);
840 for (i = 0; i < efx->n_channels; i++) {
849 rc2 = ef4_soft_enable_interrupts(efx);
852 netif_err(efx, drv, efx->net_dev,
854 ef4_schedule_reset(efx, RESET_TYPE_DISABLE);
856 ef4_start_all(efx);
857 netif_device_attach(efx->net_dev);
863 efx->rxq_entries = old_rxq_entries;
864 efx->txq_entries = old_txq_entries;
865 for (i = 0; i < efx->n_channels; i++) {
866 channel = efx->channel[i];
867 efx->channel[i] = other_channel[i];
905 void ef4_link_status_changed(struct ef4_nic *efx)
907 struct ef4_link_state *link_state = &efx->link_state;
913 if (!netif_running(efx->net_dev))
916 if (link_state->up != netif_carrier_ok(efx->net_dev)) {
917 efx->n_link_state_changes++;
920 netif_carrier_on(efx->net_dev);
922 netif_carrier_off(efx->net_dev);
927 netif_info(efx, link, efx->net_dev,
930 efx->net_dev->mtu);
932 netif_info(efx, link, efx->net_dev, "link down\n");
935 void ef4_link_set_advertising(struct ef4_nic *efx, u32 advertising)
937 efx->link_advertising = advertising;
940 efx->wanted_fc |= (EF4_FC_TX | EF4_FC_RX);
942 efx->wanted_fc &= ~(EF4_FC_TX | EF4_FC_RX);
944 efx->wanted_fc ^= EF4_FC_TX;
948 void ef4_link_set_wanted_fc(struct ef4_nic *efx, u8 wanted_fc)
950 efx->wanted_fc = wanted_fc;
951 if (efx->link_advertising) {
953 efx->link_advertising |= (ADVERTISED_Pause |
956 efx->link_advertising &= ~(ADVERTISED_Pause |
959 efx->link_advertising ^= ADVERTISED_Asym_Pause;
963 static void ef4_fini_port(struct ef4_nic *efx);
965 /* We assume that efx->type->reconfigure_mac will always try to sync RX
968 void ef4_mac_reconfigure(struct ef4_nic *efx)
970 down_read(&efx->filter_sem);
971 efx->type->reconfigure_mac(efx);
972 up_read(&efx->filter_sem);
982 int __ef4_reconfigure_port(struct ef4_nic *efx)
987 WARN_ON(!mutex_is_locked(&efx->mac_lock));
990 phy_mode = efx->phy_mode;
991 if (LOOPBACK_INTERNAL(efx))
992 efx->phy_mode |= PHY_MODE_TX_DISABLED;
994 efx->phy_mode &= ~PHY_MODE_TX_DISABLED;
996 rc = efx->type->reconfigure_port(efx);
999 efx->phy_mode = phy_mode;
1006 int ef4_reconfigure_port(struct ef4_nic *efx)
1010 EF4_ASSERT_RESET_SERIALISED(efx);
1012 mutex_lock(&efx->mac_lock);
1013 rc = __ef4_reconfigure_port(efx);
1014 mutex_unlock(&efx->mac_lock);
1024 struct ef4_nic *efx = container_of(data, struct ef4_nic, mac_work);
1026 mutex_lock(&efx->mac_lock);
1027 if (efx->port_enabled)
1028 ef4_mac_reconfigure(efx);
1029 mutex_unlock(&efx->mac_lock);
1032 static int ef4_probe_port(struct ef4_nic *efx)
1036 netif_dbg(efx, probe, efx->net_dev, "create port\n");
1039 efx->phy_mode = PHY_MODE_SPECIAL;
1042 rc = efx->type->probe_port(efx);
1047 ether_addr_copy(efx->net_dev->dev_addr, efx->net_dev->perm_addr);
1052 static int ef4_init_port(struct ef4_nic *efx)
1056 netif_dbg(efx, drv, efx->net_dev, "init port\n");
1058 mutex_lock(&efx->mac_lock);
1060 rc = efx->phy_op->init(efx);
1064 efx->port_initialized = true;
1068 ef4_mac_reconfigure(efx);
1071 rc = efx->phy_op->reconfigure(efx);
1075 mutex_unlock(&efx->mac_lock);
1079 efx->phy_op->fini(efx);
1081 mutex_unlock(&efx->mac_lock);
1085 static void ef4_start_port(struct ef4_nic *efx)
1087 netif_dbg(efx, ifup, efx->net_dev, "start port\n");
1088 BUG_ON(efx->port_enabled);
1090 mutex_lock(&efx->mac_lock);
1091 efx->port_enabled = true;
1094 ef4_mac_reconfigure(efx);
1096 mutex_unlock(&efx->mac_lock);
1104 static void ef4_stop_port(struct ef4_nic *efx)
1106 netif_dbg(efx, ifdown, efx->net_dev, "stop port\n");
1108 EF4_ASSERT_RESET_SERIALISED(efx);
1110 mutex_lock(&efx->mac_lock);
1111 efx->port_enabled = false;
1112 mutex_unlock(&efx->mac_lock);
1115 netif_addr_lock_bh(efx->net_dev);
1116 netif_addr_unlock_bh(efx->net_dev);
1118 cancel_delayed_work_sync(&efx->monitor_work);
1119 ef4_selftest_async_cancel(efx);
1120 cancel_work_sync(&efx->mac_work);
1123 static void ef4_fini_port(struct ef4_nic *efx)
1125 netif_dbg(efx, drv, efx->net_dev, "shut down port\n");
1127 if (!efx->port_initialized)
1130 efx->phy_op->fini(efx);
1131 efx->port_initialized = false;
1133 efx->link_state.up = false;
1134 ef4_link_status_changed(efx);
1137 static void ef4_remove_port(struct ef4_nic *efx)
1139 netif_dbg(efx, drv, efx->net_dev, "destroying port\n");
1141 efx->type->remove_port(efx);
1160 static void ef4_associate(struct ef4_nic *efx)
1164 if (efx->primary == efx) {
1167 netif_dbg(efx, probe, efx->net_dev, "adding to primary list\n");
1168 list_add_tail(&efx->node, &ef4_primary_list);
1172 if (ef4_same_controller(efx, other)) {
1176 pci_name(efx->pci_dev),
1177 efx->net_dev->name);
1179 &efx->secondary_list);
1180 other->primary = efx;
1187 if (ef4_same_controller(efx, other)) {
1188 netif_dbg(efx, probe, efx->net_dev,
1192 list_add_tail(&efx->node,
1194 efx->primary = other;
1199 netif_dbg(efx, probe, efx->net_dev,
1201 list_add_tail(&efx->node, &ef4_unassociated_list);
1205 static void ef4_dissociate(struct ef4_nic *efx)
1209 list_del(&efx->node);
1210 efx->primary = NULL;
1212 list_for_each_entry_safe(other, next, &efx->secondary_list, node) {
1222 static int ef4_init_io(struct ef4_nic *efx)
1224 struct pci_dev *pci_dev = efx->pci_dev;
1225 dma_addr_t dma_mask = efx->type->max_dma_mask;
1226 unsigned int mem_map_size = efx->type->mem_map_size(efx);
1229 netif_dbg(efx, probe, efx->net_dev, "initialising I/O\n");
1231 bar = efx->type->mem_bar;
1235 netif_err(efx, probe, efx->net_dev,
1253 netif_err(efx, probe, efx->net_dev,
1257 netif_dbg(efx, probe, efx->net_dev,
1260 efx->membase_phys = pci_resource_start(efx->pci_dev, bar);
1263 netif_err(efx, probe, efx->net_dev,
1268 efx->membase = ioremap(efx->membase_phys, mem_map_size);
1269 if (!efx->membase) {
1270 netif_err(efx, probe, efx->net_dev,
1272 (unsigned long long)efx->membase_phys, mem_map_size);
1276 netif_dbg(efx, probe, efx->net_dev,
1278 (unsigned long long)efx->membase_phys, mem_map_size,
1279 efx->membase);
1284 pci_release_region(efx->pci_dev, bar);
1286 efx->membase_phys = 0;
1288 pci_disable_device(efx->pci_dev);
1293 static void ef4_fini_io(struct ef4_nic *efx)
1297 netif_dbg(efx, drv, efx->net_dev, "shutting down I/O\n");
1299 if (efx->membase) {
1300 iounmap(efx->membase);
1301 efx->membase = NULL;
1304 if (efx->membase_phys) {
1305 bar = efx->type->mem_bar;
1306 pci_release_region(efx->pci_dev, bar);
1307 efx->membase_phys = 0;
1311 if (!pci_vfs_assigned(efx->pci_dev))
1312 pci_disable_device(efx->pci_dev);
1315 void ef4_set_default_rx_indir_table(struct ef4_nic *efx)
1319 for (i = 0; i < ARRAY_SIZE(efx->rx_indir_table); i++)
1320 efx->rx_indir_table[i] =
1321 ethtool_rxfh_indir_default(i, efx->rss_spread);
1324 static unsigned int ef4_wanted_parallelism(struct ef4_nic *efx)
1334 netif_warn(efx, probe, efx->net_dev,
1352 netif_cond_dbg(efx, probe, efx->net_dev, !rss_cpus, warn,
1364 static int ef4_probe_interrupts(struct ef4_nic *efx)
1371 if (efx->extra_channel_type[i])
1374 if (efx->interrupt_mode == EF4_INT_MODE_MSIX) {
1378 n_channels = ef4_wanted_parallelism(efx);
1382 n_channels = min(n_channels, efx->max_channels);
1386 rc = pci_enable_msix_range(efx->pci_dev,
1390 efx->interrupt_mode = EF4_INT_MODE_MSI;
1391 netif_err(efx, drv, efx->net_dev,
1394 netif_err(efx, drv, efx->net_dev,
1397 netif_err(efx, drv, efx->net_dev,
1403 efx->n_channels = n_channels;
1407 efx->n_tx_channels = min(max(n_channels / 2,
1409 efx->max_tx_channels);
1410 efx->n_rx_channels = max(n_channels -
1411 efx->n_tx_channels,
1414 efx->n_tx_channels = min(n_channels,
1415 efx->max_tx_channels);
1416 efx->n_rx_channels = n_channels;
1418 for (i = 0; i < efx->n_channels; i++)
1419 ef4_get_channel(efx, i)->irq =
1425 if (efx->interrupt_mode == EF4_INT_MODE_MSI) {
1426 efx->n_channels = 1;
1427 efx->n_rx_channels = 1;
1428 efx->n_tx_channels = 1;
1429 rc = pci_enable_msi(efx->pci_dev);
1431 ef4_get_channel(efx, 0)->irq = efx->pci_dev->irq;
1433 netif_err(efx, drv, efx->net_dev,
1435 efx->interrupt_mode = EF4_INT_MODE_LEGACY;
1440 if (efx->interrupt_mode == EF4_INT_MODE_LEGACY) {
1441 efx->n_channels = 1 + (ef4_separate_tx_channels ? 1 : 0);
1442 efx->n_rx_channels = 1;
1443 efx->n_tx_channels = 1;
1444 efx->legacy_irq = efx->pci_dev->irq;
1448 j = efx->n_channels;
1450 if (!efx->extra_channel_type[i])
1452 if (efx->interrupt_mode != EF4_INT_MODE_MSIX ||
1453 efx->n_channels <= extra_channels) {
1454 efx->extra_channel_type[i]->handle_no_channel(efx);
1457 ef4_get_channel(efx, j)->type =
1458 efx->extra_channel_type[i];
1462 efx->rss_spread = efx->n_rx_channels;
1467 static int ef4_soft_enable_interrupts(struct ef4_nic *efx)
1472 BUG_ON(efx->state == STATE_DISABLED);
1474 efx->irq_soft_enabled = true;
1477 ef4_for_each_channel(channel, efx) {
1489 ef4_for_each_channel(channel, efx) {
1500 static void ef4_soft_disable_interrupts(struct ef4_nic *efx)
1504 if (efx->state == STATE_DISABLED)
1507 efx->irq_soft_enabled = false;
1510 if (efx->legacy_irq)
1511 synchronize_irq(efx->legacy_irq);
1513 ef4_for_each_channel(channel, efx) {
1523 static int ef4_enable_interrupts(struct ef4_nic *efx)
1528 BUG_ON(efx->state == STATE_DISABLED);
1530 if (efx->eeh_disabled_legacy_irq) {
1531 enable_irq(efx->legacy_irq);
1532 efx->eeh_disabled_legacy_irq = false;
1535 efx->type->irq_enable_master(efx);
1537 ef4_for_each_channel(channel, efx) {
1545 rc = ef4_soft_enable_interrupts(efx);
1553 ef4_for_each_channel(channel, efx) {
1560 efx->type->irq_disable_non_ev(efx);
1565 static void ef4_disable_interrupts(struct ef4_nic *efx)
1569 ef4_soft_disable_interrupts(efx);
1571 ef4_for_each_channel(channel, efx) {
1576 efx->type->irq_disable_non_ev(efx);
1579 static void ef4_remove_interrupts(struct ef4_nic *efx)
1584 ef4_for_each_channel(channel, efx)
1586 pci_disable_msi(efx->pci_dev);
1587 pci_disable_msix(efx->pci_dev);
1590 efx->legacy_irq = 0;
1593 static void ef4_set_channels(struct ef4_nic *efx)
1598 efx->tx_channel_offset =
1600 efx->n_channels - efx->n_tx_channels : 0;
1606 ef4_for_each_channel(channel, efx) {
1607 if (channel->channel < efx->n_rx_channels)
1613 tx_queue->queue -= (efx->tx_channel_offset *
1618 static int ef4_probe_nic(struct ef4_nic *efx)
1622 netif_dbg(efx, probe, efx->net_dev, "creating NIC\n");
1625 rc = efx->type->probe(efx);
1630 if (!efx->max_channels || !efx->max_tx_channels) {
1631 netif_err(efx, drv, efx->net_dev,
1641 rc = ef4_probe_interrupts(efx);
1645 ef4_set_channels(efx);
1648 rc = efx->type->dimension_resources(efx);
1654 ef4_remove_interrupts(efx);
1658 if (efx->n_channels > 1)
1659 netdev_rss_key_fill(&efx->rx_hash_key,
1660 sizeof(efx->rx_hash_key));
1661 ef4_set_default_rx_indir_table(efx);
1663 netif_set_real_num_tx_queues(efx->net_dev, efx->n_tx_channels);
1664 netif_set_real_num_rx_queues(efx->net_dev, efx->n_rx_channels);
1667 efx->irq_mod_step_us = DIV_ROUND_UP(efx->timer_quantum_ns, 1000);
1668 ef4_init_irq_moderation(efx, tx_irq_mod_usec, rx_irq_mod_usec, true,
1674 ef4_remove_interrupts(efx);
1676 efx->type->remove(efx);
1680 static void ef4_remove_nic(struct ef4_nic *efx)
1682 netif_dbg(efx, drv, efx->net_dev, "destroying NIC\n");
1684 ef4_remove_interrupts(efx);
1685 efx->type->remove(efx);
1688 static int ef4_probe_filters(struct ef4_nic *efx)
1692 spin_lock_init(&efx->filter_lock);
1693 init_rwsem(&efx->filter_sem);
1694 mutex_lock(&efx->mac_lock);
1695 down_write(&efx->filter_sem);
1696 rc = efx->type->filter_table_probe(efx);
1701 if (efx->type->offload_features & NETIF_F_NTUPLE) {
1705 ef4_for_each_channel(channel, efx) {
1707 kcalloc(efx->type->max_rx_ip_filters,
1714 i < efx->type->max_rx_ip_filters;
1721 ef4_for_each_channel(channel, efx)
1723 efx->type->filter_table_remove(efx);
1728 efx->rps_expire_index = efx->rps_expire_channel = 0;
1732 up_write(&efx->filter_sem);
1733 mutex_unlock(&efx->mac_lock);
1737 static void ef4_remove_filters(struct ef4_nic *efx)
1742 ef4_for_each_channel(channel, efx)
1745 down_write(&efx->filter_sem);
1746 efx->type->filter_table_remove(efx);
1747 up_write(&efx->filter_sem);
1750 static void ef4_restore_filters(struct ef4_nic *efx)
1752 down_read(&efx->filter_sem);
1753 efx->type->filter_table_restore(efx);
1754 up_read(&efx->filter_sem);
1763 static int ef4_probe_all(struct ef4_nic *efx)
1767 rc = ef4_probe_nic(efx);
1769 netif_err(efx, probe, efx->net_dev, "failed to create NIC\n");
1773 rc = ef4_probe_port(efx);
1775 netif_err(efx, probe, efx->net_dev, "failed to create port\n");
1780 if (WARN_ON(EF4_DEFAULT_DMAQ_SIZE < EF4_TXQ_MIN_ENT(efx))) {
1784 efx->rxq_entries = efx->txq_entries = EF4_DEFAULT_DMAQ_SIZE;
1786 rc = ef4_probe_filters(efx);
1788 netif_err(efx, probe, efx->net_dev,
1793 rc = ef4_probe_channels(efx);
1800 ef4_remove_filters(efx);
1803 ef4_remove_port(efx);
1805 ef4_remove_nic(efx);
1817 static void ef4_start_all(struct ef4_nic *efx)
1819 EF4_ASSERT_RESET_SERIALISED(efx);
1820 BUG_ON(efx->state == STATE_DISABLED);
1824 if (efx->port_enabled || !netif_running(efx->net_dev) ||
1825 efx->reset_pending)
1828 ef4_start_port(efx);
1829 ef4_start_datapath(efx);
1832 if (efx->type->monitor != NULL)
1833 queue_delayed_work(efx->workqueue, &efx->monitor_work,
1836 efx->type->start_stats(efx);
1837 efx->type->pull_stats(efx);
1838 spin_lock_bh(&efx->stats_lock);
1839 efx->type->update_stats(efx, NULL, NULL);
1840 spin_unlock_bh(&efx->stats_lock);
1848 static void ef4_stop_all(struct ef4_nic *efx)
1850 EF4_ASSERT_RESET_SERIALISED(efx);
1853 if (!efx->port_enabled)
1859 efx->type->pull_stats(efx);
1860 spin_lock_bh(&efx->stats_lock);
1861 efx->type->update_stats(efx, NULL, NULL);
1862 spin_unlock_bh(&efx->stats_lock);
1863 efx->type->stop_stats(efx);
1864 ef4_stop_port(efx);
1870 WARN_ON(netif_running(efx->net_dev) &&
1871 netif_device_present(efx->net_dev));
1872 netif_tx_disable(efx->net_dev);
1874 ef4_stop_datapath(efx);
1877 static void ef4_remove_all(struct ef4_nic *efx)
1879 ef4_remove_channels(efx);
1880 ef4_remove_filters(efx);
1881 ef4_remove_port(efx);
1882 ef4_remove_nic(efx);
1890 unsigned int ef4_usecs_to_ticks(struct ef4_nic *efx, unsigned int usecs)
1894 if (usecs * 1000 < efx->timer_quantum_ns)
1896 return usecs * 1000 / efx->timer_quantum_ns;
1899 unsigned int ef4_ticks_to_usecs(struct ef4_nic *efx, unsigned int ticks)
1904 return DIV_ROUND_UP(ticks * efx->timer_quantum_ns, 1000);
1908 int ef4_init_irq_moderation(struct ef4_nic *efx, unsigned int tx_usecs,
1915 EF4_ASSERT_RESET_SERIALISED(efx);
1917 timer_max_us = efx->timer_max_ns / 1000;
1922 if (tx_usecs != rx_usecs && efx->tx_channel_offset == 0 &&
1924 netif_err(efx, drv, efx->net_dev, "Channels are shared. "
1929 efx->irq_rx_adaptive = rx_adaptive;
1930 efx->irq_rx_moderation_us = rx_usecs;
1931 ef4_for_each_channel(channel, efx) {
1941 void ef4_get_irq_moderation(struct ef4_nic *efx, unsigned int *tx_usecs,
1944 *rx_adaptive = efx->irq_rx_adaptive;
1945 *rx_usecs = efx->irq_rx_moderation_us;
1951 if (efx->tx_channel_offset == 0) {
1956 tx_channel = efx->channel[efx->tx_channel_offset];
1970 struct ef4_nic *efx = container_of(data, struct ef4_nic,
1973 netif_vdbg(efx, timer, efx->net_dev,
1976 BUG_ON(efx->type->monitor == NULL);
1981 if (mutex_trylock(&efx->mac_lock)) {
1982 if (efx->port_enabled)
1983 efx->type->monitor(efx);
1984 mutex_unlock(&efx->mac_lock);
1987 queue_delayed_work(efx->workqueue, &efx->monitor_work,
2002 struct ef4_nic *efx = netdev_priv(net_dev);
2010 return mdio_mii_ioctl(&efx->mdio, data, cmd);
2021 struct ef4_nic *efx = channel->efx;
2023 channel->napi_dev = efx->net_dev;
2028 static void ef4_init_napi(struct ef4_nic *efx)
2032 ef4_for_each_channel(channel, efx)
2044 static void ef4_fini_napi(struct ef4_nic *efx)
2048 ef4_for_each_channel(channel, efx)
2061 struct ef4_nic *efx = netdev_priv(net_dev);
2064 netif_dbg(efx, ifup, efx->net_dev, "opening device on CPU %d\n",
2067 rc = ef4_check_disabled(efx);
2070 if (efx->phy_mode & PHY_MODE_SPECIAL)
2075 ef4_link_status_changed(efx);
2077 ef4_start_all(efx);
2078 ef4_selftest_async_start(efx);
2088 struct ef4_nic *efx = netdev_priv(net_dev);
2090 netif_dbg(efx, ifdown, efx->net_dev, "closing on CPU %d\n",
2094 ef4_stop_all(efx);
2103 struct ef4_nic *efx = netdev_priv(net_dev);
2105 spin_lock_bh(&efx->stats_lock);
2106 efx->type->update_stats(efx, NULL, stats);
2107 spin_unlock_bh(&efx->stats_lock);
2113 struct ef4_nic *efx = netdev_priv(net_dev);
2115 netif_err(efx, tx_err, efx->net_dev,
2117 efx->port_enabled);
2119 ef4_schedule_reset(efx, RESET_TYPE_TX_WATCHDOG);
2126 struct ef4_nic *efx = netdev_priv(net_dev);
2129 rc = ef4_check_disabled(efx);
2133 netif_dbg(efx, drv, efx->net_dev, "changing MTU to %d\n", new_mtu);
2135 ef4_device_detach_sync(efx);
2136 ef4_stop_all(efx);
2138 mutex_lock(&efx->mac_lock);
2140 ef4_mac_reconfigure(efx);
2141 mutex_unlock(&efx->mac_lock);
2143 ef4_start_all(efx);
2144 netif_device_attach(efx->net_dev);
2150 struct ef4_nic *efx = netdev_priv(net_dev);
2157 netif_err(efx, drv, efx->net_dev,
2166 if (efx->type->set_mac_address) {
2167 rc = efx->type->set_mac_address(efx);
2175 mutex_lock(&efx->mac_lock);
2176 ef4_mac_reconfigure(efx);
2177 mutex_unlock(&efx->mac_lock);
2185 struct ef4_nic *efx = netdev_priv(net_dev);
2187 if (efx->port_enabled)
2188 queue_work(efx->workqueue, &efx->mac_work);
2194 struct ef4_nic *efx = netdev_priv(net_dev);
2199 rc = efx->type->filter_clear_rx(efx, EF4_FILTER_PRI_MANUAL);
2233 static void ef4_update_name(struct ef4_nic *efx)
2235 strcpy(efx->name, efx->net_dev->name);
2236 ef4_mtd_rename(efx);
2237 ef4_set_channel_names(efx);
2259 struct ef4_nic *efx = dev_get_drvdata(dev);
2260 return sprintf(buf, "%d\n", efx->phy_type);
2264 static int ef4_register_netdev(struct ef4_nic *efx)
2266 struct net_device *net_dev = efx->net_dev;
2271 net_dev->irq = efx->pci_dev->irq;
2284 efx->state = STATE_READY;
2286 if (efx->reset_pending) {
2287 netif_err(efx, probe, efx->net_dev,
2296 ef4_update_name(efx);
2305 ef4_for_each_channel(channel, efx) {
2311 ef4_associate(efx);
2315 rc = device_create_file(&efx->pci_dev->dev, &dev_attr_phy_type);
2317 netif_err(efx, drv, efx->net_dev,
2325 ef4_dissociate(efx);
2328 efx->state = STATE_UNINIT;
2330 netif_err(efx, drv, efx->net_dev, "could not register net dev\n");
2334 static void ef4_unregister_netdev(struct ef4_nic *efx)
2336 if (!efx->net_dev)
2339 BUG_ON(netdev_priv(efx->net_dev) != efx);
2341 if (ef4_dev_registered(efx)) {
2342 strlcpy(efx->name, pci_name(efx->pci_dev), sizeof(efx->name));
2343 device_remove_file(&efx->pci_dev->dev, &dev_attr_phy_type);
2344 unregister_netdev(efx->net_dev);
2356 void ef4_reset_down(struct ef4_nic *efx, enum reset_type method)
2358 EF4_ASSERT_RESET_SERIALISED(efx);
2360 ef4_stop_all(efx);
2361 ef4_disable_interrupts(efx);
2363 mutex_lock(&efx->mac_lock);
2364 if (efx->port_initialized && method != RESET_TYPE_INVISIBLE &&
2366 efx->phy_op->fini(efx);
2367 efx->type->fini(efx);
2375 int ef4_reset_up(struct ef4_nic *efx, enum reset_type method, bool ok)
2379 EF4_ASSERT_RESET_SERIALISED(efx);
2382 rc = efx->type->init(efx);
2384 netif_err(efx, drv, efx->net_dev, "failed to initialise NIC\n");
2391 if (efx->port_initialized && method != RESET_TYPE_INVISIBLE &&
2393 rc = efx->phy_op->init(efx);
2396 rc = efx->phy_op->reconfigure(efx);
2398 netif_err(efx, drv, efx->net_dev,
2402 rc = ef4_enable_interrupts(efx);
2406 down_read(&efx->filter_sem);
2407 ef4_restore_filters(efx);
2408 up_read(&efx->filter_sem);
2410 mutex_unlock(&efx->mac_lock);
2412 ef4_start_all(efx);
2417 efx->port_initialized = false;
2419 mutex_unlock(&efx->mac_lock);
2429 int ef4_reset(struct ef4_nic *efx, enum reset_type method)
2434 netif_info(efx, drv, efx->net_dev, "resetting (%s)\n",
2437 ef4_device_detach_sync(efx);
2438 ef4_reset_down(efx, method);
2440 rc = efx->type->reset(efx, method);
2442 netif_err(efx, drv, efx->net_dev, "failed to reset hardware\n");
2450 efx->reset_pending &= -(1 << (method + 1));
2452 __clear_bit(method, &efx->reset_pending);
2458 pci_set_master(efx->pci_dev);
2465 rc2 = ef4_reset_up(efx, method, !disabled);
2473 dev_close(efx->net_dev);
2474 netif_err(efx, drv, efx->net_dev, "has been disabled\n");
2475 efx->state = STATE_DISABLED;
2477 netif_dbg(efx, drv, efx->net_dev, "reset complete\n");
2478 netif_device_attach(efx->net_dev);
2488 int ef4_try_recovery(struct ef4_nic *efx)
2496 struct eeh_dev *eehdev = pci_dev_to_eeh_dev(efx->pci_dev);
2512 struct ef4_nic *efx = container_of(data, struct ef4_nic, reset_work);
2516 pending = READ_ONCE(efx->reset_pending);
2521 ef4_try_recovery(efx))
2533 if (efx->state == STATE_READY)
2534 (void)ef4_reset(efx, method);
2539 void ef4_schedule_reset(struct ef4_nic *efx, enum reset_type type)
2543 if (efx->state == STATE_RECOVERY) {
2544 netif_dbg(efx, drv, efx->net_dev,
2559 netif_dbg(efx, drv, efx->net_dev, "scheduling %s reset\n",
2563 method = efx->type->map_reset_reason(type);
2564 netif_dbg(efx, drv, efx->net_dev,
2570 set_bit(method, &efx->reset_pending);
2576 if (READ_ONCE(efx->state) != STATE_READY)
2579 queue_work(reset_workqueue, &efx->reset_work);
2608 int ef4_port_dummy_op_int(struct ef4_nic *efx)
2612 void ef4_port_dummy_op_void(struct ef4_nic *efx) {}
2614 static bool ef4_port_dummy_op_poll(struct ef4_nic *efx)
2635 static int ef4_init_struct(struct ef4_nic *efx,
2641 INIT_LIST_HEAD(&efx->node);
2642 INIT_LIST_HEAD(&efx->secondary_list);
2643 spin_lock_init(&efx->biu_lock);
2645 INIT_LIST_HEAD(&efx->mtd_list);
2647 INIT_WORK(&efx->reset_work, ef4_reset_work);
2648 INIT_DELAYED_WORK(&efx->monitor_work, ef4_monitor);
2649 INIT_DELAYED_WORK(&efx->selftest_work, ef4_selftest_async_work);
2650 efx->pci_dev = pci_dev;
2651 efx->msg_enable = debug;
2652 efx->state = STATE_UNINIT;
2653 strlcpy(efx->name, pci_name(pci_dev), sizeof(efx->name));
2655 efx->net_dev = net_dev;
2656 efx->rx_prefix_size = efx->type->rx_prefix_size;
2657 efx->rx_ip_align =
2658 NET_IP_ALIGN ? (efx->rx_prefix_size + NET_IP_ALIGN) % 4 : 0;
2659 efx->rx_packet_hash_offset =
2660 efx->type->rx_hash_offset - efx->type->rx_prefix_size;
2661 efx->rx_packet_ts_offset =
2662 efx->type->rx_ts_offset - efx->type->rx_prefix_size;
2663 spin_lock_init(&efx->stats_lock);
2664 mutex_init(&efx->mac_lock);
2665 efx->phy_op = &ef4_dummy_phy_operations;
2666 efx->mdio.dev = net_dev;
2667 INIT_WORK(&efx->mac_work, ef4_mac_work);
2668 init_waitqueue_head(&efx->flush_wq);
2671 efx->channel[i] = ef4_alloc_channel(efx, i, NULL);
2672 if (!efx->channel[i])
2674 efx->msi_context[i].efx = efx;
2675 efx->msi_context[i].index = i;
2679 efx->interrupt_mode = max(efx->type->max_interrupt_mode,
2683 snprintf(efx->workqueue_name, sizeof(efx->workqueue_name), "sfc%s",
2685 efx->workqueue = create_singlethread_workqueue(efx->workqueue_name);
2686 if (!efx->workqueue)
2692 ef4_fini_struct(efx);
2696 static void ef4_fini_struct(struct ef4_nic *efx)
2701 kfree(efx->channel[i]);
2703 kfree(efx->vpd_sn);
2705 if (efx->workqueue) {
2706 destroy_workqueue(efx->workqueue);
2707 efx->workqueue = NULL;
2711 void ef4_update_sw_stats(struct ef4_nic *efx, u64 *stats)
2716 ef4_for_each_channel(channel, efx)
2719 stats[GENERIC_STAT_rx_noskb_drops] = atomic_read(&efx->n_rx_noskb_drops);
2731 static void ef4_pci_remove_main(struct ef4_nic *efx)
2736 BUG_ON(efx->state == STATE_READY);
2737 cancel_work_sync(&efx->reset_work);
2739 ef4_disable_interrupts(efx);
2740 ef4_nic_fini_interrupt(efx);
2741 ef4_fini_port(efx);
2742 efx->type->fini(efx);
2743 ef4_fini_napi(efx);
2744 ef4_remove_all(efx);
2753 struct ef4_nic *efx;
2755 efx = pci_get_drvdata(pci_dev);
2756 if (!efx)
2761 ef4_dissociate(efx);
2762 dev_close(efx->net_dev);
2763 ef4_disable_interrupts(efx);
2764 efx->state = STATE_UNINIT;
2767 ef4_unregister_netdev(efx);
2769 ef4_mtd_remove(efx);
2771 ef4_pci_remove_main(efx);
2773 ef4_fini_io(efx);
2774 netif_dbg(efx, drv, efx->net_dev, "shutdown successful\n");
2776 ef4_fini_struct(efx);
2777 free_netdev(efx->net_dev);
2788 static void ef4_probe_vpd_strings(struct ef4_nic *efx)
2790 struct pci_dev *dev = efx->pci_dev;
2798 netif_err(efx, drv, efx->net_dev, "Unable to read VPD\n");
2805 netif_err(efx, drv, efx->net_dev, "VPD Read-only not found\n");
2818 netif_err(efx, drv, efx->net_dev, "Part number not found\n");
2825 netif_err(efx, drv, efx->net_dev, "Incomplete part number\n");
2829 netif_info(efx, drv, efx->net_dev,
2836 netif_err(efx, drv, efx->net_dev, "Serial number not found\n");
2843 netif_err(efx, drv, efx->net_dev, "Incomplete serial number\n");
2847 efx->vpd_sn = kmalloc(j + 1, GFP_KERNEL);
2848 if (!efx->vpd_sn)
2851 snprintf(efx->vpd_sn, j + 1, "%s", &vpd_data[i]);
2858 static int ef4_pci_probe_main(struct ef4_nic *efx)
2863 rc = ef4_probe_all(efx);
2867 ef4_init_napi(efx);
2869 rc = efx->type->init(efx);
2871 netif_err(efx, probe, efx->net_dev,
2876 rc = ef4_init_port(efx);
2878 netif_err(efx, probe, efx->net_dev,
2883 rc = ef4_nic_init_interrupt(efx);
2886 rc = ef4_enable_interrupts(efx);
2893 ef4_nic_fini_interrupt(efx);
2895 ef4_fini_port(efx);
2897 efx->type->fini(efx);
2899 ef4_fini_napi(efx);
2900 ef4_remove_all(efx);
2918 struct ef4_nic *efx;
2922 net_dev = alloc_etherdev_mqs(sizeof(*efx), EF4_MAX_CORE_TX_QUEUES,
2926 efx = netdev_priv(net_dev);
2927 efx->type = (const struct ef4_nic_type *) entry->driver_data;
2928 efx->fixed_features |= NETIF_F_HIGHDMA;
2930 pci_set_drvdata(pci_dev, efx);
2932 rc = ef4_init_struct(efx, pci_dev, net_dev);
2936 netif_info(efx, probe, efx->net_dev,
2939 ef4_probe_vpd_strings(efx);
2942 rc = ef4_init_io(efx);
2946 rc = ef4_pci_probe_main(efx);
2950 net_dev->features |= (efx->type->offload_features | NETIF_F_SG |
2956 net_dev->hw_features = net_dev->features & ~efx->fixed_features;
2963 net_dev->features |= efx->fixed_features;
2965 rc = ef4_register_netdev(efx);
2969 netif_dbg(efx, probe, efx->net_dev, "initialisation successful\n");
2973 rc = ef4_mtd_probe(efx);
2976 netif_warn(efx, probe, efx->net_dev,
2981 netif_notice(efx, probe, efx->net_dev,
2988 ef4_pci_remove_main(efx);
2990 ef4_fini_io(efx);
2992 ef4_fini_struct(efx);
2995 netif_dbg(efx, drv, efx->net_dev, "initialisation failed. rc=%d\n", rc);
3002 struct ef4_nic *efx = dev_get_drvdata(dev);
3006 if (efx->state != STATE_DISABLED) {
3007 efx->state = STATE_UNINIT;
3009 ef4_device_detach_sync(efx);
3011 ef4_stop_all(efx);
3012 ef4_disable_interrupts(efx);
3023 struct ef4_nic *efx = dev_get_drvdata(dev);
3027 if (efx->state != STATE_DISABLED) {
3028 rc = ef4_enable_interrupts(efx);
3032 mutex_lock(&efx->mac_lock);
3033 efx->phy_op->reconfigure(efx);
3034 mutex_unlock(&efx->mac_lock);
3036 ef4_start_all(efx);
3038 netif_device_attach(efx->net_dev);
3040 efx->state = STATE_READY;
3042 efx->type->resume_wol(efx);
3048 queue_work(reset_workqueue, &efx->reset_work);
3061 struct ef4_nic *efx = pci_get_drvdata(pci_dev);
3063 efx->type->fini(efx);
3065 efx->reset_pending = 0;
3075 struct ef4_nic *efx = pci_get_drvdata(pci_dev);
3085 pci_set_master(efx->pci_dev);
3086 rc = efx->type->reset(efx, RESET_TYPE_ALL);
3089 rc = efx->type->init(efx);
3124 struct ef4_nic *efx = pci_get_drvdata(pdev);
3131 if (efx->state != STATE_DISABLED) {
3132 efx->state = STATE_RECOVERY;
3133 efx->reset_pending = 0;
3135 ef4_device_detach_sync(efx);
3137 ef4_stop_all(efx);
3138 ef4_disable_interrupts(efx);
3158 struct ef4_nic *efx = pci_get_drvdata(pdev);
3162 netif_err(efx, hw, efx->net_dev,
3173 struct ef4_nic *efx = pci_get_drvdata(pdev);
3178 if (efx->state == STATE_DISABLED)
3181 rc = ef4_reset(efx, RESET_TYPE_ALL);
3183 netif_err(efx, hw, efx->net_dev,
3186 efx->state = STATE_READY;
3187 netif_dbg(efx, hw, efx->net_dev,