1// SPDX-License-Identifier: GPL-2.0 2/* Copyright (c) 2019 Intel Corporation */ 3 4#include "igc.h" 5 6#include <linux/module.h> 7#include <linux/device.h> 8#include <linux/pci.h> 9#include <linux/ptp_classify.h> 10#include <linux/clocksource.h> 11#include <linux/ktime.h> 12 13#define INCVALUE_MASK 0x7fffffff 14#define ISGN 0x80000000 15 16#define IGC_SYSTIM_OVERFLOW_PERIOD (HZ * 60 * 9) 17#define IGC_PTP_TX_TIMEOUT (HZ * 15) 18 19/* SYSTIM read access for I225 */ 20void igc_ptp_read(struct igc_adapter *adapter, struct timespec64 *ts) 21{ 22 struct igc_hw *hw = &adapter->hw; 23 u32 sec, nsec; 24 25 /* The timestamp is latched when SYSTIML is read. */ 26 nsec = rd32(IGC_SYSTIML); 27 sec = rd32(IGC_SYSTIMH); 28 29 ts->tv_sec = sec; 30 ts->tv_nsec = nsec; 31} 32 33static void igc_ptp_write_i225(struct igc_adapter *adapter, 34 const struct timespec64 *ts) 35{ 36 struct igc_hw *hw = &adapter->hw; 37 38 wr32(IGC_SYSTIML, ts->tv_nsec); 39 wr32(IGC_SYSTIMH, ts->tv_sec); 40} 41 42static int igc_ptp_adjfine_i225(struct ptp_clock_info *ptp, long scaled_ppm) 43{ 44 struct igc_adapter *igc = container_of(ptp, struct igc_adapter, 45 ptp_caps); 46 struct igc_hw *hw = &igc->hw; 47 int neg_adj = 0; 48 u64 rate; 49 u32 inca; 50 51 if (scaled_ppm < 0) { 52 neg_adj = 1; 53 scaled_ppm = -scaled_ppm; 54 } 55 rate = scaled_ppm; 56 rate <<= 14; 57 rate = div_u64(rate, 78125); 58 59 inca = rate & INCVALUE_MASK; 60 if (neg_adj) 61 inca |= ISGN; 62 63 wr32(IGC_TIMINCA, inca); 64 65 return 0; 66} 67 68static int igc_ptp_adjtime_i225(struct ptp_clock_info *ptp, s64 delta) 69{ 70 struct igc_adapter *igc = container_of(ptp, struct igc_adapter, 71 ptp_caps); 72 struct timespec64 now, then = ns_to_timespec64(delta); 73 unsigned long flags; 74 75 spin_lock_irqsave(&igc->tmreg_lock, flags); 76 77 igc_ptp_read(igc, &now); 78 now = timespec64_add(now, then); 79 igc_ptp_write_i225(igc, (const struct timespec64 *)&now); 80 81 spin_unlock_irqrestore(&igc->tmreg_lock, flags); 82 83 return 0; 84} 85 86static int igc_ptp_gettimex64_i225(struct ptp_clock_info *ptp, 87 struct timespec64 *ts, 88 struct ptp_system_timestamp *sts) 89{ 90 struct igc_adapter *igc = container_of(ptp, struct igc_adapter, 91 ptp_caps); 92 struct igc_hw *hw = &igc->hw; 93 unsigned long flags; 94 95 spin_lock_irqsave(&igc->tmreg_lock, flags); 96 97 ptp_read_system_prets(sts); 98 ts->tv_nsec = rd32(IGC_SYSTIML); 99 ts->tv_sec = rd32(IGC_SYSTIMH); 100 ptp_read_system_postts(sts); 101 102 spin_unlock_irqrestore(&igc->tmreg_lock, flags); 103 104 return 0; 105} 106 107static int igc_ptp_settime_i225(struct ptp_clock_info *ptp, 108 const struct timespec64 *ts) 109{ 110 struct igc_adapter *igc = container_of(ptp, struct igc_adapter, 111 ptp_caps); 112 unsigned long flags; 113 114 spin_lock_irqsave(&igc->tmreg_lock, flags); 115 116 igc_ptp_write_i225(igc, ts); 117 118 spin_unlock_irqrestore(&igc->tmreg_lock, flags); 119 120 return 0; 121} 122 123static int igc_ptp_feature_enable_i225(struct ptp_clock_info *ptp, 124 struct ptp_clock_request *rq, int on) 125{ 126 return -EOPNOTSUPP; 127} 128 129/** 130 * igc_ptp_systim_to_hwtstamp - convert system time value to HW timestamp 131 * @adapter: board private structure 132 * @hwtstamps: timestamp structure to update 133 * @systim: unsigned 64bit system time value 134 * 135 * We need to convert the system time value stored in the RX/TXSTMP registers 136 * into a hwtstamp which can be used by the upper level timestamping functions. 137 * 138 * Returns 0 on success. 139 **/ 140static int igc_ptp_systim_to_hwtstamp(struct igc_adapter *adapter, 141 struct skb_shared_hwtstamps *hwtstamps, 142 u64 systim) 143{ 144 switch (adapter->hw.mac.type) { 145 case igc_i225: 146 memset(hwtstamps, 0, sizeof(*hwtstamps)); 147 /* Upper 32 bits contain s, lower 32 bits contain ns. */ 148 hwtstamps->hwtstamp = ktime_set(systim >> 32, 149 systim & 0xFFFFFFFF); 150 break; 151 default: 152 return -EINVAL; 153 } 154 return 0; 155} 156 157/** 158 * igc_ptp_rx_pktstamp - Retrieve timestamp from Rx packet buffer 159 * @q_vector: Pointer to interrupt specific structure 160 * @va: Pointer to address containing Rx buffer 161 * @skb: Buffer containing timestamp and packet 162 * 163 * This function retrieves the timestamp saved in the beginning of packet 164 * buffer. While two timestamps are available, one in timer0 reference and the 165 * other in timer1 reference, this function considers only the timestamp in 166 * timer0 reference. 167 */ 168void igc_ptp_rx_pktstamp(struct igc_q_vector *q_vector, __le32 *va, 169 struct sk_buff *skb) 170{ 171 struct igc_adapter *adapter = q_vector->adapter; 172 u64 regval; 173 int adjust; 174 175 /* Timestamps are saved in little endian at the beginning of the packet 176 * buffer following the layout: 177 * 178 * DWORD: | 0 | 1 | 2 | 3 | 179 * Field: | Timer1 SYSTIML | Timer1 SYSTIMH | Timer0 SYSTIML | Timer0 SYSTIMH | 180 * 181 * SYSTIML holds the nanoseconds part while SYSTIMH holds the seconds 182 * part of the timestamp. 183 */ 184 regval = le32_to_cpu(va[2]); 185 regval |= (u64)le32_to_cpu(va[3]) << 32; 186 igc_ptp_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), regval); 187 188 /* Adjust timestamp for the RX latency based on link speed */ 189 switch (adapter->link_speed) { 190 case SPEED_10: 191 adjust = IGC_I225_RX_LATENCY_10; 192 break; 193 case SPEED_100: 194 adjust = IGC_I225_RX_LATENCY_100; 195 break; 196 case SPEED_1000: 197 adjust = IGC_I225_RX_LATENCY_1000; 198 break; 199 case SPEED_2500: 200 adjust = IGC_I225_RX_LATENCY_2500; 201 break; 202 default: 203 adjust = 0; 204 netdev_warn_once(adapter->netdev, "Imprecise timestamp\n"); 205 break; 206 } 207 skb_hwtstamps(skb)->hwtstamp = 208 ktime_sub_ns(skb_hwtstamps(skb)->hwtstamp, adjust); 209} 210 211static void igc_ptp_disable_rx_timestamp(struct igc_adapter *adapter) 212{ 213 struct igc_hw *hw = &adapter->hw; 214 u32 val; 215 int i; 216 217 wr32(IGC_TSYNCRXCTL, 0); 218 219 for (i = 0; i < adapter->num_rx_queues; i++) { 220 val = rd32(IGC_SRRCTL(i)); 221 val &= ~IGC_SRRCTL_TIMESTAMP; 222 wr32(IGC_SRRCTL(i), val); 223 } 224 225 val = rd32(IGC_RXPBS); 226 val &= ~IGC_RXPBS_CFG_TS_EN; 227 wr32(IGC_RXPBS, val); 228} 229 230static void igc_ptp_enable_rx_timestamp(struct igc_adapter *adapter) 231{ 232 struct igc_hw *hw = &adapter->hw; 233 u32 val; 234 int i; 235 236 val = rd32(IGC_RXPBS); 237 val |= IGC_RXPBS_CFG_TS_EN; 238 wr32(IGC_RXPBS, val); 239 240 for (i = 0; i < adapter->num_rx_queues; i++) { 241 val = rd32(IGC_SRRCTL(i)); 242 /* FIXME: For now, only support retrieving RX timestamps from 243 * timer 0. 244 */ 245 val |= IGC_SRRCTL_TIMER1SEL(0) | IGC_SRRCTL_TIMER0SEL(0) | 246 IGC_SRRCTL_TIMESTAMP; 247 wr32(IGC_SRRCTL(i), val); 248 } 249 250 val = IGC_TSYNCRXCTL_ENABLED | IGC_TSYNCRXCTL_TYPE_ALL | 251 IGC_TSYNCRXCTL_RXSYNSIG; 252 wr32(IGC_TSYNCRXCTL, val); 253} 254 255static void igc_ptp_disable_tx_timestamp(struct igc_adapter *adapter) 256{ 257 struct igc_hw *hw = &adapter->hw; 258 259 wr32(IGC_TSYNCTXCTL, 0); 260} 261 262static void igc_ptp_enable_tx_timestamp(struct igc_adapter *adapter) 263{ 264 struct igc_hw *hw = &adapter->hw; 265 266 wr32(IGC_TSYNCTXCTL, IGC_TSYNCTXCTL_ENABLED | IGC_TSYNCTXCTL_TXSYNSIG); 267 268 /* Read TXSTMP registers to discard any timestamp previously stored. */ 269 rd32(IGC_TXSTMPL); 270 rd32(IGC_TXSTMPH); 271} 272 273/** 274 * igc_ptp_set_timestamp_mode - setup hardware for timestamping 275 * @adapter: networking device structure 276 * @config: hwtstamp configuration 277 * 278 * Return: 0 in case of success, negative errno code otherwise. 279 */ 280static int igc_ptp_set_timestamp_mode(struct igc_adapter *adapter, 281 struct hwtstamp_config *config) 282{ 283 /* reserved for future extensions */ 284 if (config->flags) 285 return -EINVAL; 286 287 switch (config->tx_type) { 288 case HWTSTAMP_TX_OFF: 289 igc_ptp_disable_tx_timestamp(adapter); 290 break; 291 case HWTSTAMP_TX_ON: 292 igc_ptp_enable_tx_timestamp(adapter); 293 break; 294 default: 295 return -ERANGE; 296 } 297 298 switch (config->rx_filter) { 299 case HWTSTAMP_FILTER_NONE: 300 igc_ptp_disable_rx_timestamp(adapter); 301 break; 302 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: 303 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: 304 case HWTSTAMP_FILTER_PTP_V2_EVENT: 305 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT: 306 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: 307 case HWTSTAMP_FILTER_PTP_V2_SYNC: 308 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC: 309 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: 310 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: 311 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ: 312 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: 313 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: 314 case HWTSTAMP_FILTER_NTP_ALL: 315 case HWTSTAMP_FILTER_ALL: 316 igc_ptp_enable_rx_timestamp(adapter); 317 config->rx_filter = HWTSTAMP_FILTER_ALL; 318 break; 319 default: 320 return -ERANGE; 321 } 322 323 return 0; 324} 325 326/* Requires adapter->ptp_tx_lock held by caller. */ 327static void igc_ptp_tx_timeout(struct igc_adapter *adapter) 328{ 329 struct igc_hw *hw = &adapter->hw; 330 331 dev_kfree_skb_any(adapter->ptp_tx_skb); 332 adapter->ptp_tx_skb = NULL; 333 adapter->tx_hwtstamp_timeouts++; 334 /* Clear the tx valid bit in TSYNCTXCTL register to enable interrupt. */ 335 rd32(IGC_TXSTMPH); 336 netdev_warn(adapter->netdev, "Tx timestamp timeout\n"); 337} 338 339void igc_ptp_tx_hang(struct igc_adapter *adapter) 340{ 341 unsigned long flags; 342 343 spin_lock_irqsave(&adapter->ptp_tx_lock, flags); 344 345 if (!adapter->ptp_tx_skb) 346 goto unlock; 347 348 if (time_is_after_jiffies(adapter->ptp_tx_start + IGC_PTP_TX_TIMEOUT)) 349 goto unlock; 350 351 igc_ptp_tx_timeout(adapter); 352 353unlock: 354 spin_unlock_irqrestore(&adapter->ptp_tx_lock, flags); 355} 356 357/** 358 * igc_ptp_tx_hwtstamp - utility function which checks for TX time stamp 359 * @adapter: Board private structure 360 * 361 * If we were asked to do hardware stamping and such a time stamp is 362 * available, then it must have been for this skb here because we only 363 * allow only one such packet into the queue. 364 * 365 * Context: Expects adapter->ptp_tx_lock to be held by caller. 366 */ 367static void igc_ptp_tx_hwtstamp(struct igc_adapter *adapter) 368{ 369 struct sk_buff *skb = adapter->ptp_tx_skb; 370 struct skb_shared_hwtstamps shhwtstamps; 371 struct igc_hw *hw = &adapter->hw; 372 int adjust = 0; 373 u64 regval; 374 375 if (WARN_ON_ONCE(!skb)) 376 return; 377 378 regval = rd32(IGC_TXSTMPL); 379 regval |= (u64)rd32(IGC_TXSTMPH) << 32; 380 if (igc_ptp_systim_to_hwtstamp(adapter, &shhwtstamps, regval)) 381 return; 382 383 switch (adapter->link_speed) { 384 case SPEED_10: 385 adjust = IGC_I225_TX_LATENCY_10; 386 break; 387 case SPEED_100: 388 adjust = IGC_I225_TX_LATENCY_100; 389 break; 390 case SPEED_1000: 391 adjust = IGC_I225_TX_LATENCY_1000; 392 break; 393 case SPEED_2500: 394 adjust = IGC_I225_TX_LATENCY_2500; 395 break; 396 } 397 398 shhwtstamps.hwtstamp = 399 ktime_add_ns(shhwtstamps.hwtstamp, adjust); 400 401 adapter->ptp_tx_skb = NULL; 402 403 /* Notify the stack and free the skb after we've unlocked */ 404 skb_tstamp_tx(skb, &shhwtstamps); 405 dev_kfree_skb_any(skb); 406} 407 408/** 409 * igc_ptp_tx_work 410 * @work: pointer to work struct 411 * 412 * This work function checks the TSYNCTXCTL valid bit to determine when 413 * a timestamp has been taken for the current stored skb. 414 */ 415static void igc_ptp_tx_work(struct work_struct *work) 416{ 417 struct igc_adapter *adapter = container_of(work, struct igc_adapter, 418 ptp_tx_work); 419 struct igc_hw *hw = &adapter->hw; 420 unsigned long flags; 421 u32 tsynctxctl; 422 423 spin_lock_irqsave(&adapter->ptp_tx_lock, flags); 424 425 if (!adapter->ptp_tx_skb) 426 goto unlock; 427 428 tsynctxctl = rd32(IGC_TSYNCTXCTL); 429 tsynctxctl &= IGC_TSYNCTXCTL_TXTT_0; 430 if (!tsynctxctl) { 431 WARN_ONCE(1, "Received a TSTAMP interrupt but no TSTAMP is ready.\n"); 432 goto unlock; 433 } 434 435 igc_ptp_tx_hwtstamp(adapter); 436 437unlock: 438 spin_unlock_irqrestore(&adapter->ptp_tx_lock, flags); 439} 440 441/** 442 * igc_ptp_set_ts_config - set hardware time stamping config 443 * @netdev: network interface device structure 444 * @ifr: interface request data 445 * 446 **/ 447int igc_ptp_set_ts_config(struct net_device *netdev, struct ifreq *ifr) 448{ 449 struct igc_adapter *adapter = netdev_priv(netdev); 450 struct hwtstamp_config config; 451 int err; 452 453 if (copy_from_user(&config, ifr->ifr_data, sizeof(config))) 454 return -EFAULT; 455 456 err = igc_ptp_set_timestamp_mode(adapter, &config); 457 if (err) 458 return err; 459 460 /* save these settings for future reference */ 461 memcpy(&adapter->tstamp_config, &config, 462 sizeof(adapter->tstamp_config)); 463 464 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ? 465 -EFAULT : 0; 466} 467 468/** 469 * igc_ptp_get_ts_config - get hardware time stamping config 470 * @netdev: network interface device structure 471 * @ifr: interface request data 472 * 473 * Get the hwtstamp_config settings to return to the user. Rather than attempt 474 * to deconstruct the settings from the registers, just return a shadow copy 475 * of the last known settings. 476 **/ 477int igc_ptp_get_ts_config(struct net_device *netdev, struct ifreq *ifr) 478{ 479 struct igc_adapter *adapter = netdev_priv(netdev); 480 struct hwtstamp_config *config = &adapter->tstamp_config; 481 482 return copy_to_user(ifr->ifr_data, config, sizeof(*config)) ? 483 -EFAULT : 0; 484} 485 486/** 487 * igc_ptp_init - Initialize PTP functionality 488 * @adapter: Board private structure 489 * 490 * This function is called at device probe to initialize the PTP 491 * functionality. 492 */ 493void igc_ptp_init(struct igc_adapter *adapter) 494{ 495 struct net_device *netdev = adapter->netdev; 496 struct igc_hw *hw = &adapter->hw; 497 498 switch (hw->mac.type) { 499 case igc_i225: 500 snprintf(adapter->ptp_caps.name, 16, "%pm", netdev->dev_addr); 501 adapter->ptp_caps.owner = THIS_MODULE; 502 adapter->ptp_caps.max_adj = 62499999; 503 adapter->ptp_caps.adjfine = igc_ptp_adjfine_i225; 504 adapter->ptp_caps.adjtime = igc_ptp_adjtime_i225; 505 adapter->ptp_caps.gettimex64 = igc_ptp_gettimex64_i225; 506 adapter->ptp_caps.settime64 = igc_ptp_settime_i225; 507 adapter->ptp_caps.enable = igc_ptp_feature_enable_i225; 508 break; 509 default: 510 adapter->ptp_clock = NULL; 511 return; 512 } 513 514 spin_lock_init(&adapter->ptp_tx_lock); 515 spin_lock_init(&adapter->tmreg_lock); 516 INIT_WORK(&adapter->ptp_tx_work, igc_ptp_tx_work); 517 518 adapter->tstamp_config.rx_filter = HWTSTAMP_FILTER_NONE; 519 adapter->tstamp_config.tx_type = HWTSTAMP_TX_OFF; 520 521 adapter->prev_ptp_time = ktime_to_timespec64(ktime_get_real()); 522 adapter->ptp_reset_start = ktime_get(); 523 524 adapter->ptp_clock = ptp_clock_register(&adapter->ptp_caps, 525 &adapter->pdev->dev); 526 if (IS_ERR(adapter->ptp_clock)) { 527 adapter->ptp_clock = NULL; 528 netdev_err(netdev, "ptp_clock_register failed\n"); 529 } else if (adapter->ptp_clock) { 530 netdev_info(netdev, "PHC added\n"); 531 adapter->ptp_flags |= IGC_PTP_ENABLED; 532 } 533} 534 535static void igc_ptp_time_save(struct igc_adapter *adapter) 536{ 537 igc_ptp_read(adapter, &adapter->prev_ptp_time); 538 adapter->ptp_reset_start = ktime_get(); 539} 540 541static void igc_ptp_time_restore(struct igc_adapter *adapter) 542{ 543 struct timespec64 ts = adapter->prev_ptp_time; 544 ktime_t delta; 545 546 delta = ktime_sub(ktime_get(), adapter->ptp_reset_start); 547 548 timespec64_add_ns(&ts, ktime_to_ns(delta)); 549 550 igc_ptp_write_i225(adapter, &ts); 551} 552 553/** 554 * igc_ptp_suspend - Disable PTP work items and prepare for suspend 555 * @adapter: Board private structure 556 * 557 * This function stops the overflow check work and PTP Tx timestamp work, and 558 * will prepare the device for OS suspend. 559 */ 560void igc_ptp_suspend(struct igc_adapter *adapter) 561{ 562 if (!(adapter->ptp_flags & IGC_PTP_ENABLED)) 563 return; 564 565 cancel_work_sync(&adapter->ptp_tx_work); 566 dev_kfree_skb_any(adapter->ptp_tx_skb); 567 adapter->ptp_tx_skb = NULL; 568 569 if (pci_device_is_present(adapter->pdev)) 570 igc_ptp_time_save(adapter); 571} 572 573/** 574 * igc_ptp_stop - Disable PTP device and stop the overflow check. 575 * @adapter: Board private structure. 576 * 577 * This function stops the PTP support and cancels the delayed work. 578 **/ 579void igc_ptp_stop(struct igc_adapter *adapter) 580{ 581 igc_ptp_suspend(adapter); 582 583 if (adapter->ptp_clock) { 584 ptp_clock_unregister(adapter->ptp_clock); 585 netdev_info(adapter->netdev, "PHC removed\n"); 586 adapter->ptp_flags &= ~IGC_PTP_ENABLED; 587 } 588} 589 590/** 591 * igc_ptp_reset - Re-enable the adapter for PTP following a reset. 592 * @adapter: Board private structure. 593 * 594 * This function handles the reset work required to re-enable the PTP device. 595 **/ 596void igc_ptp_reset(struct igc_adapter *adapter) 597{ 598 struct igc_hw *hw = &adapter->hw; 599 unsigned long flags; 600 601 /* reset the tstamp_config */ 602 igc_ptp_set_timestamp_mode(adapter, &adapter->tstamp_config); 603 604 spin_lock_irqsave(&adapter->tmreg_lock, flags); 605 606 switch (adapter->hw.mac.type) { 607 case igc_i225: 608 wr32(IGC_TSAUXC, 0x0); 609 wr32(IGC_TSSDP, 0x0); 610 wr32(IGC_TSIM, IGC_TSICR_INTERRUPTS); 611 wr32(IGC_IMS, IGC_IMS_TS); 612 break; 613 default: 614 /* No work to do. */ 615 goto out; 616 } 617 618 /* Re-initialize the timer. */ 619 if (hw->mac.type == igc_i225) { 620 igc_ptp_time_restore(adapter); 621 } else { 622 timecounter_init(&adapter->tc, &adapter->cc, 623 ktime_to_ns(ktime_get_real())); 624 } 625out: 626 spin_unlock_irqrestore(&adapter->tmreg_lock, flags); 627 628 wrfl(); 629} 630