1// SPDX-License-Identifier: (GPL-2.0 OR MIT) 2/* Google virtual Ethernet (gve) driver 3 * 4 * Copyright (C) 2015-2021 Google, Inc. 5 */ 6 7#include <linux/etherdevice.h> 8#include <linux/pci.h> 9#include "gve.h" 10#include "gve_adminq.h" 11#include "gve_register.h" 12 13#define GVE_MAX_ADMINQ_RELEASE_CHECK 500 14#define GVE_ADMINQ_SLEEP_LEN 20 15#define GVE_MAX_ADMINQ_EVENT_COUNTER_CHECK 100 16 17#define GVE_DEVICE_OPTION_ERROR_FMT "%s option error:\n" \ 18"Expected: length=%d, feature_mask=%x.\n" \ 19"Actual: length=%d, feature_mask=%x.\n" 20 21#define GVE_DEVICE_OPTION_TOO_BIG_FMT "Length of %s option larger than expected. Possible older version of guest driver.\n" 22 23static 24struct gve_device_option *gve_get_next_option(struct gve_device_descriptor *descriptor, 25 struct gve_device_option *option) 26{ 27 void *option_end, *descriptor_end; 28 29 option_end = (void *)(option + 1) + be16_to_cpu(option->option_length); 30 descriptor_end = (void *)descriptor + be16_to_cpu(descriptor->total_length); 31 32 return option_end > descriptor_end ? NULL : (struct gve_device_option *)option_end; 33} 34 35static 36void gve_parse_device_option(struct gve_priv *priv, 37 struct gve_device_descriptor *device_descriptor, 38 struct gve_device_option *option, 39 struct gve_device_option_gqi_rda **dev_op_gqi_rda, 40 struct gve_device_option_gqi_qpl **dev_op_gqi_qpl, 41 struct gve_device_option_dqo_rda **dev_op_dqo_rda, 42 struct gve_device_option_jumbo_frames **dev_op_jumbo_frames, 43 struct gve_device_option_dqo_qpl **dev_op_dqo_qpl) 44{ 45 u32 req_feat_mask = be32_to_cpu(option->required_features_mask); 46 u16 option_length = be16_to_cpu(option->option_length); 47 u16 option_id = be16_to_cpu(option->option_id); 48 49 /* If the length or feature mask doesn't match, continue without 50 * enabling the feature. 51 */ 52 switch (option_id) { 53 case GVE_DEV_OPT_ID_GQI_RAW_ADDRESSING: 54 if (option_length != GVE_DEV_OPT_LEN_GQI_RAW_ADDRESSING || 55 req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_GQI_RAW_ADDRESSING) { 56 dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT, 57 "Raw Addressing", 58 GVE_DEV_OPT_LEN_GQI_RAW_ADDRESSING, 59 GVE_DEV_OPT_REQ_FEAT_MASK_GQI_RAW_ADDRESSING, 60 option_length, req_feat_mask); 61 break; 62 } 63 64 dev_info(&priv->pdev->dev, 65 "Gqi raw addressing device option enabled.\n"); 66 priv->queue_format = GVE_GQI_RDA_FORMAT; 67 break; 68 case GVE_DEV_OPT_ID_GQI_RDA: 69 if (option_length < sizeof(**dev_op_gqi_rda) || 70 req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_GQI_RDA) { 71 dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT, 72 "GQI RDA", (int)sizeof(**dev_op_gqi_rda), 73 GVE_DEV_OPT_REQ_FEAT_MASK_GQI_RDA, 74 option_length, req_feat_mask); 75 break; 76 } 77 78 if (option_length > sizeof(**dev_op_gqi_rda)) { 79 dev_warn(&priv->pdev->dev, 80 GVE_DEVICE_OPTION_TOO_BIG_FMT, "GQI RDA"); 81 } 82 *dev_op_gqi_rda = (void *)(option + 1); 83 break; 84 case GVE_DEV_OPT_ID_GQI_QPL: 85 if (option_length < sizeof(**dev_op_gqi_qpl) || 86 req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_GQI_QPL) { 87 dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT, 88 "GQI QPL", (int)sizeof(**dev_op_gqi_qpl), 89 GVE_DEV_OPT_REQ_FEAT_MASK_GQI_QPL, 90 option_length, req_feat_mask); 91 break; 92 } 93 94 if (option_length > sizeof(**dev_op_gqi_qpl)) { 95 dev_warn(&priv->pdev->dev, 96 GVE_DEVICE_OPTION_TOO_BIG_FMT, "GQI QPL"); 97 } 98 *dev_op_gqi_qpl = (void *)(option + 1); 99 break; 100 case GVE_DEV_OPT_ID_DQO_RDA: 101 if (option_length < sizeof(**dev_op_dqo_rda) || 102 req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_DQO_RDA) { 103 dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT, 104 "DQO RDA", (int)sizeof(**dev_op_dqo_rda), 105 GVE_DEV_OPT_REQ_FEAT_MASK_DQO_RDA, 106 option_length, req_feat_mask); 107 break; 108 } 109 110 if (option_length > sizeof(**dev_op_dqo_rda)) { 111 dev_warn(&priv->pdev->dev, 112 GVE_DEVICE_OPTION_TOO_BIG_FMT, "DQO RDA"); 113 } 114 *dev_op_dqo_rda = (void *)(option + 1); 115 break; 116 case GVE_DEV_OPT_ID_DQO_QPL: 117 if (option_length < sizeof(**dev_op_dqo_qpl) || 118 req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_DQO_QPL) { 119 dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT, 120 "DQO QPL", (int)sizeof(**dev_op_dqo_qpl), 121 GVE_DEV_OPT_REQ_FEAT_MASK_DQO_QPL, 122 option_length, req_feat_mask); 123 break; 124 } 125 126 if (option_length > sizeof(**dev_op_dqo_qpl)) { 127 dev_warn(&priv->pdev->dev, 128 GVE_DEVICE_OPTION_TOO_BIG_FMT, "DQO QPL"); 129 } 130 *dev_op_dqo_qpl = (void *)(option + 1); 131 break; 132 case GVE_DEV_OPT_ID_JUMBO_FRAMES: 133 if (option_length < sizeof(**dev_op_jumbo_frames) || 134 req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_JUMBO_FRAMES) { 135 dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT, 136 "Jumbo Frames", 137 (int)sizeof(**dev_op_jumbo_frames), 138 GVE_DEV_OPT_REQ_FEAT_MASK_JUMBO_FRAMES, 139 option_length, req_feat_mask); 140 break; 141 } 142 143 if (option_length > sizeof(**dev_op_jumbo_frames)) { 144 dev_warn(&priv->pdev->dev, 145 GVE_DEVICE_OPTION_TOO_BIG_FMT, 146 "Jumbo Frames"); 147 } 148 *dev_op_jumbo_frames = (void *)(option + 1); 149 break; 150 default: 151 /* If we don't recognize the option just continue 152 * without doing anything. 153 */ 154 dev_dbg(&priv->pdev->dev, "Unrecognized device option 0x%hx not enabled.\n", 155 option_id); 156 } 157} 158 159/* Process all device options for a given describe device call. */ 160static int 161gve_process_device_options(struct gve_priv *priv, 162 struct gve_device_descriptor *descriptor, 163 struct gve_device_option_gqi_rda **dev_op_gqi_rda, 164 struct gve_device_option_gqi_qpl **dev_op_gqi_qpl, 165 struct gve_device_option_dqo_rda **dev_op_dqo_rda, 166 struct gve_device_option_jumbo_frames **dev_op_jumbo_frames, 167 struct gve_device_option_dqo_qpl **dev_op_dqo_qpl) 168{ 169 const int num_options = be16_to_cpu(descriptor->num_device_options); 170 struct gve_device_option *dev_opt; 171 int i; 172 173 /* The options struct directly follows the device descriptor. */ 174 dev_opt = (void *)(descriptor + 1); 175 for (i = 0; i < num_options; i++) { 176 struct gve_device_option *next_opt; 177 178 next_opt = gve_get_next_option(descriptor, dev_opt); 179 if (!next_opt) { 180 dev_err(&priv->dev->dev, 181 "options exceed device_descriptor's total length.\n"); 182 return -EINVAL; 183 } 184 185 gve_parse_device_option(priv, descriptor, dev_opt, 186 dev_op_gqi_rda, dev_op_gqi_qpl, 187 dev_op_dqo_rda, dev_op_jumbo_frames, 188 dev_op_dqo_qpl); 189 dev_opt = next_opt; 190 } 191 192 return 0; 193} 194 195int gve_adminq_alloc(struct device *dev, struct gve_priv *priv) 196{ 197 priv->adminq = dma_alloc_coherent(dev, PAGE_SIZE, 198 &priv->adminq_bus_addr, GFP_KERNEL); 199 if (unlikely(!priv->adminq)) 200 return -ENOMEM; 201 202 priv->adminq_mask = (PAGE_SIZE / sizeof(union gve_adminq_command)) - 1; 203 priv->adminq_prod_cnt = 0; 204 priv->adminq_cmd_fail = 0; 205 priv->adminq_timeouts = 0; 206 priv->adminq_describe_device_cnt = 0; 207 priv->adminq_cfg_device_resources_cnt = 0; 208 priv->adminq_register_page_list_cnt = 0; 209 priv->adminq_unregister_page_list_cnt = 0; 210 priv->adminq_create_tx_queue_cnt = 0; 211 priv->adminq_create_rx_queue_cnt = 0; 212 priv->adminq_destroy_tx_queue_cnt = 0; 213 priv->adminq_destroy_rx_queue_cnt = 0; 214 priv->adminq_dcfg_device_resources_cnt = 0; 215 priv->adminq_set_driver_parameter_cnt = 0; 216 priv->adminq_report_stats_cnt = 0; 217 priv->adminq_report_link_speed_cnt = 0; 218 priv->adminq_get_ptype_map_cnt = 0; 219 220 /* Setup Admin queue with the device */ 221 iowrite32be(priv->adminq_bus_addr / PAGE_SIZE, 222 &priv->reg_bar0->adminq_pfn); 223 224 gve_set_admin_queue_ok(priv); 225 return 0; 226} 227 228void gve_adminq_release(struct gve_priv *priv) 229{ 230 int i = 0; 231 232 /* Tell the device the adminq is leaving */ 233 iowrite32be(0x0, &priv->reg_bar0->adminq_pfn); 234 while (ioread32be(&priv->reg_bar0->adminq_pfn)) { 235 /* If this is reached the device is unrecoverable and still 236 * holding memory. Continue looping to avoid memory corruption, 237 * but WARN so it is visible what is going on. 238 */ 239 if (i == GVE_MAX_ADMINQ_RELEASE_CHECK) 240 WARN(1, "Unrecoverable platform error!"); 241 i++; 242 msleep(GVE_ADMINQ_SLEEP_LEN); 243 } 244 gve_clear_device_rings_ok(priv); 245 gve_clear_device_resources_ok(priv); 246 gve_clear_admin_queue_ok(priv); 247} 248 249void gve_adminq_free(struct device *dev, struct gve_priv *priv) 250{ 251 if (!gve_get_admin_queue_ok(priv)) 252 return; 253 gve_adminq_release(priv); 254 dma_free_coherent(dev, PAGE_SIZE, priv->adminq, priv->adminq_bus_addr); 255 gve_clear_admin_queue_ok(priv); 256} 257 258static void gve_adminq_kick_cmd(struct gve_priv *priv, u32 prod_cnt) 259{ 260 iowrite32be(prod_cnt, &priv->reg_bar0->adminq_doorbell); 261} 262 263static bool gve_adminq_wait_for_cmd(struct gve_priv *priv, u32 prod_cnt) 264{ 265 int i; 266 267 for (i = 0; i < GVE_MAX_ADMINQ_EVENT_COUNTER_CHECK; i++) { 268 if (ioread32be(&priv->reg_bar0->adminq_event_counter) 269 == prod_cnt) 270 return true; 271 msleep(GVE_ADMINQ_SLEEP_LEN); 272 } 273 274 return false; 275} 276 277static int gve_adminq_parse_err(struct gve_priv *priv, u32 status) 278{ 279 if (status != GVE_ADMINQ_COMMAND_PASSED && 280 status != GVE_ADMINQ_COMMAND_UNSET) { 281 dev_err(&priv->pdev->dev, "AQ command failed with status %d\n", status); 282 priv->adminq_cmd_fail++; 283 } 284 switch (status) { 285 case GVE_ADMINQ_COMMAND_PASSED: 286 return 0; 287 case GVE_ADMINQ_COMMAND_UNSET: 288 dev_err(&priv->pdev->dev, "parse_aq_err: err and status both unset, this should not be possible.\n"); 289 return -EINVAL; 290 case GVE_ADMINQ_COMMAND_ERROR_ABORTED: 291 case GVE_ADMINQ_COMMAND_ERROR_CANCELLED: 292 case GVE_ADMINQ_COMMAND_ERROR_DATALOSS: 293 case GVE_ADMINQ_COMMAND_ERROR_FAILED_PRECONDITION: 294 case GVE_ADMINQ_COMMAND_ERROR_UNAVAILABLE: 295 return -EAGAIN; 296 case GVE_ADMINQ_COMMAND_ERROR_ALREADY_EXISTS: 297 case GVE_ADMINQ_COMMAND_ERROR_INTERNAL_ERROR: 298 case GVE_ADMINQ_COMMAND_ERROR_INVALID_ARGUMENT: 299 case GVE_ADMINQ_COMMAND_ERROR_NOT_FOUND: 300 case GVE_ADMINQ_COMMAND_ERROR_OUT_OF_RANGE: 301 case GVE_ADMINQ_COMMAND_ERROR_UNKNOWN_ERROR: 302 return -EINVAL; 303 case GVE_ADMINQ_COMMAND_ERROR_DEADLINE_EXCEEDED: 304 return -ETIME; 305 case GVE_ADMINQ_COMMAND_ERROR_PERMISSION_DENIED: 306 case GVE_ADMINQ_COMMAND_ERROR_UNAUTHENTICATED: 307 return -EACCES; 308 case GVE_ADMINQ_COMMAND_ERROR_RESOURCE_EXHAUSTED: 309 return -ENOMEM; 310 case GVE_ADMINQ_COMMAND_ERROR_UNIMPLEMENTED: 311 return -EOPNOTSUPP; 312 default: 313 dev_err(&priv->pdev->dev, "parse_aq_err: unknown status code %d\n", status); 314 return -EINVAL; 315 } 316} 317 318/* Flushes all AQ commands currently queued and waits for them to complete. 319 * If there are failures, it will return the first error. 320 */ 321static int gve_adminq_kick_and_wait(struct gve_priv *priv) 322{ 323 int tail, head; 324 int i; 325 326 tail = ioread32be(&priv->reg_bar0->adminq_event_counter); 327 head = priv->adminq_prod_cnt; 328 329 gve_adminq_kick_cmd(priv, head); 330 if (!gve_adminq_wait_for_cmd(priv, head)) { 331 dev_err(&priv->pdev->dev, "AQ commands timed out, need to reset AQ\n"); 332 priv->adminq_timeouts++; 333 return -ENOTRECOVERABLE; 334 } 335 336 for (i = tail; i < head; i++) { 337 union gve_adminq_command *cmd; 338 u32 status, err; 339 340 cmd = &priv->adminq[i & priv->adminq_mask]; 341 status = be32_to_cpu(READ_ONCE(cmd->status)); 342 err = gve_adminq_parse_err(priv, status); 343 if (err) 344 // Return the first error if we failed. 345 return err; 346 } 347 348 return 0; 349} 350 351/* This function is not threadsafe - the caller is responsible for any 352 * necessary locks. 353 */ 354static int gve_adminq_issue_cmd(struct gve_priv *priv, 355 union gve_adminq_command *cmd_orig) 356{ 357 union gve_adminq_command *cmd; 358 u32 opcode; 359 u32 tail; 360 361 tail = ioread32be(&priv->reg_bar0->adminq_event_counter); 362 363 // Check if next command will overflow the buffer. 364 if (((priv->adminq_prod_cnt + 1) & priv->adminq_mask) == 365 (tail & priv->adminq_mask)) { 366 int err; 367 368 // Flush existing commands to make room. 369 err = gve_adminq_kick_and_wait(priv); 370 if (err) 371 return err; 372 373 // Retry. 374 tail = ioread32be(&priv->reg_bar0->adminq_event_counter); 375 if (((priv->adminq_prod_cnt + 1) & priv->adminq_mask) == 376 (tail & priv->adminq_mask)) { 377 // This should never happen. We just flushed the 378 // command queue so there should be enough space. 379 return -ENOMEM; 380 } 381 } 382 383 cmd = &priv->adminq[priv->adminq_prod_cnt & priv->adminq_mask]; 384 priv->adminq_prod_cnt++; 385 386 memcpy(cmd, cmd_orig, sizeof(*cmd_orig)); 387 opcode = be32_to_cpu(READ_ONCE(cmd->opcode)); 388 389 switch (opcode) { 390 case GVE_ADMINQ_DESCRIBE_DEVICE: 391 priv->adminq_describe_device_cnt++; 392 break; 393 case GVE_ADMINQ_CONFIGURE_DEVICE_RESOURCES: 394 priv->adminq_cfg_device_resources_cnt++; 395 break; 396 case GVE_ADMINQ_REGISTER_PAGE_LIST: 397 priv->adminq_register_page_list_cnt++; 398 break; 399 case GVE_ADMINQ_UNREGISTER_PAGE_LIST: 400 priv->adminq_unregister_page_list_cnt++; 401 break; 402 case GVE_ADMINQ_CREATE_TX_QUEUE: 403 priv->adminq_create_tx_queue_cnt++; 404 break; 405 case GVE_ADMINQ_CREATE_RX_QUEUE: 406 priv->adminq_create_rx_queue_cnt++; 407 break; 408 case GVE_ADMINQ_DESTROY_TX_QUEUE: 409 priv->adminq_destroy_tx_queue_cnt++; 410 break; 411 case GVE_ADMINQ_DESTROY_RX_QUEUE: 412 priv->adminq_destroy_rx_queue_cnt++; 413 break; 414 case GVE_ADMINQ_DECONFIGURE_DEVICE_RESOURCES: 415 priv->adminq_dcfg_device_resources_cnt++; 416 break; 417 case GVE_ADMINQ_SET_DRIVER_PARAMETER: 418 priv->adminq_set_driver_parameter_cnt++; 419 break; 420 case GVE_ADMINQ_REPORT_STATS: 421 priv->adminq_report_stats_cnt++; 422 break; 423 case GVE_ADMINQ_REPORT_LINK_SPEED: 424 priv->adminq_report_link_speed_cnt++; 425 break; 426 case GVE_ADMINQ_GET_PTYPE_MAP: 427 priv->adminq_get_ptype_map_cnt++; 428 break; 429 case GVE_ADMINQ_VERIFY_DRIVER_COMPATIBILITY: 430 priv->adminq_verify_driver_compatibility_cnt++; 431 break; 432 default: 433 dev_err(&priv->pdev->dev, "unknown AQ command opcode %d\n", opcode); 434 } 435 436 return 0; 437} 438 439/* This function is not threadsafe - the caller is responsible for any 440 * necessary locks. 441 * The caller is also responsible for making sure there are no commands 442 * waiting to be executed. 443 */ 444static int gve_adminq_execute_cmd(struct gve_priv *priv, 445 union gve_adminq_command *cmd_orig) 446{ 447 u32 tail, head; 448 int err; 449 450 tail = ioread32be(&priv->reg_bar0->adminq_event_counter); 451 head = priv->adminq_prod_cnt; 452 if (tail != head) 453 // This is not a valid path 454 return -EINVAL; 455 456 err = gve_adminq_issue_cmd(priv, cmd_orig); 457 if (err) 458 return err; 459 460 return gve_adminq_kick_and_wait(priv); 461} 462 463/* The device specifies that the management vector can either be the first irq 464 * or the last irq. ntfy_blk_msix_base_idx indicates the first irq assigned to 465 * the ntfy blks. It if is 0 then the management vector is last, if it is 1 then 466 * the management vector is first. 467 * 468 * gve arranges the msix vectors so that the management vector is last. 469 */ 470#define GVE_NTFY_BLK_BASE_MSIX_IDX 0 471int gve_adminq_configure_device_resources(struct gve_priv *priv, 472 dma_addr_t counter_array_bus_addr, 473 u32 num_counters, 474 dma_addr_t db_array_bus_addr, 475 u32 num_ntfy_blks) 476{ 477 union gve_adminq_command cmd; 478 479 memset(&cmd, 0, sizeof(cmd)); 480 cmd.opcode = cpu_to_be32(GVE_ADMINQ_CONFIGURE_DEVICE_RESOURCES); 481 cmd.configure_device_resources = 482 (struct gve_adminq_configure_device_resources) { 483 .counter_array = cpu_to_be64(counter_array_bus_addr), 484 .num_counters = cpu_to_be32(num_counters), 485 .irq_db_addr = cpu_to_be64(db_array_bus_addr), 486 .num_irq_dbs = cpu_to_be32(num_ntfy_blks), 487 .irq_db_stride = cpu_to_be32(sizeof(*priv->irq_db_indices)), 488 .ntfy_blk_msix_base_idx = 489 cpu_to_be32(GVE_NTFY_BLK_BASE_MSIX_IDX), 490 .queue_format = priv->queue_format, 491 }; 492 493 return gve_adminq_execute_cmd(priv, &cmd); 494} 495 496int gve_adminq_deconfigure_device_resources(struct gve_priv *priv) 497{ 498 union gve_adminq_command cmd; 499 500 memset(&cmd, 0, sizeof(cmd)); 501 cmd.opcode = cpu_to_be32(GVE_ADMINQ_DECONFIGURE_DEVICE_RESOURCES); 502 503 return gve_adminq_execute_cmd(priv, &cmd); 504} 505 506static int gve_adminq_create_tx_queue(struct gve_priv *priv, u32 queue_index) 507{ 508 struct gve_tx_ring *tx = &priv->tx[queue_index]; 509 union gve_adminq_command cmd; 510 511 memset(&cmd, 0, sizeof(cmd)); 512 cmd.opcode = cpu_to_be32(GVE_ADMINQ_CREATE_TX_QUEUE); 513 cmd.create_tx_queue = (struct gve_adminq_create_tx_queue) { 514 .queue_id = cpu_to_be32(queue_index), 515 .queue_resources_addr = 516 cpu_to_be64(tx->q_resources_bus), 517 .tx_ring_addr = cpu_to_be64(tx->bus), 518 .ntfy_id = cpu_to_be32(tx->ntfy_id), 519 }; 520 521 if (gve_is_gqi(priv)) { 522 u32 qpl_id = priv->queue_format == GVE_GQI_RDA_FORMAT ? 523 GVE_RAW_ADDRESSING_QPL_ID : tx->tx_fifo.qpl->id; 524 525 cmd.create_tx_queue.queue_page_list_id = cpu_to_be32(qpl_id); 526 } else { 527 u16 comp_ring_size; 528 u32 qpl_id = 0; 529 530 if (priv->queue_format == GVE_DQO_RDA_FORMAT) { 531 qpl_id = GVE_RAW_ADDRESSING_QPL_ID; 532 comp_ring_size = 533 priv->options_dqo_rda.tx_comp_ring_entries; 534 } else { 535 qpl_id = tx->dqo.qpl->id; 536 comp_ring_size = priv->tx_desc_cnt; 537 } 538 cmd.create_tx_queue.queue_page_list_id = cpu_to_be32(qpl_id); 539 cmd.create_tx_queue.tx_ring_size = 540 cpu_to_be16(priv->tx_desc_cnt); 541 cmd.create_tx_queue.tx_comp_ring_addr = 542 cpu_to_be64(tx->complq_bus_dqo); 543 cmd.create_tx_queue.tx_comp_ring_size = 544 cpu_to_be16(comp_ring_size); 545 } 546 547 return gve_adminq_issue_cmd(priv, &cmd); 548} 549 550int gve_adminq_create_tx_queues(struct gve_priv *priv, u32 start_id, u32 num_queues) 551{ 552 int err; 553 int i; 554 555 for (i = start_id; i < start_id + num_queues; i++) { 556 err = gve_adminq_create_tx_queue(priv, i); 557 if (err) 558 return err; 559 } 560 561 return gve_adminq_kick_and_wait(priv); 562} 563 564static int gve_adminq_create_rx_queue(struct gve_priv *priv, u32 queue_index) 565{ 566 struct gve_rx_ring *rx = &priv->rx[queue_index]; 567 union gve_adminq_command cmd; 568 569 memset(&cmd, 0, sizeof(cmd)); 570 cmd.opcode = cpu_to_be32(GVE_ADMINQ_CREATE_RX_QUEUE); 571 cmd.create_rx_queue = (struct gve_adminq_create_rx_queue) { 572 .queue_id = cpu_to_be32(queue_index), 573 .ntfy_id = cpu_to_be32(rx->ntfy_id), 574 .queue_resources_addr = cpu_to_be64(rx->q_resources_bus), 575 }; 576 577 if (gve_is_gqi(priv)) { 578 u32 qpl_id = priv->queue_format == GVE_GQI_RDA_FORMAT ? 579 GVE_RAW_ADDRESSING_QPL_ID : rx->data.qpl->id; 580 581 cmd.create_rx_queue.rx_desc_ring_addr = 582 cpu_to_be64(rx->desc.bus), 583 cmd.create_rx_queue.rx_data_ring_addr = 584 cpu_to_be64(rx->data.data_bus), 585 cmd.create_rx_queue.index = cpu_to_be32(queue_index); 586 cmd.create_rx_queue.queue_page_list_id = cpu_to_be32(qpl_id); 587 cmd.create_rx_queue.packet_buffer_size = cpu_to_be16(rx->packet_buffer_size); 588 } else { 589 u16 rx_buff_ring_entries; 590 u32 qpl_id = 0; 591 592 if (priv->queue_format == GVE_DQO_RDA_FORMAT) { 593 qpl_id = GVE_RAW_ADDRESSING_QPL_ID; 594 rx_buff_ring_entries = 595 priv->options_dqo_rda.rx_buff_ring_entries; 596 } else { 597 qpl_id = rx->dqo.qpl->id; 598 rx_buff_ring_entries = priv->rx_desc_cnt; 599 } 600 cmd.create_rx_queue.queue_page_list_id = cpu_to_be32(qpl_id); 601 cmd.create_rx_queue.rx_ring_size = 602 cpu_to_be16(priv->rx_desc_cnt); 603 cmd.create_rx_queue.rx_desc_ring_addr = 604 cpu_to_be64(rx->dqo.complq.bus); 605 cmd.create_rx_queue.rx_data_ring_addr = 606 cpu_to_be64(rx->dqo.bufq.bus); 607 cmd.create_rx_queue.packet_buffer_size = 608 cpu_to_be16(priv->data_buffer_size_dqo); 609 cmd.create_rx_queue.rx_buff_ring_size = 610 cpu_to_be16(rx_buff_ring_entries); 611 cmd.create_rx_queue.enable_rsc = 612 !!(priv->dev->features & NETIF_F_LRO); 613 } 614 615 return gve_adminq_issue_cmd(priv, &cmd); 616} 617 618int gve_adminq_create_rx_queues(struct gve_priv *priv, u32 num_queues) 619{ 620 int err; 621 int i; 622 623 for (i = 0; i < num_queues; i++) { 624 err = gve_adminq_create_rx_queue(priv, i); 625 if (err) 626 return err; 627 } 628 629 return gve_adminq_kick_and_wait(priv); 630} 631 632static int gve_adminq_destroy_tx_queue(struct gve_priv *priv, u32 queue_index) 633{ 634 union gve_adminq_command cmd; 635 int err; 636 637 memset(&cmd, 0, sizeof(cmd)); 638 cmd.opcode = cpu_to_be32(GVE_ADMINQ_DESTROY_TX_QUEUE); 639 cmd.destroy_tx_queue = (struct gve_adminq_destroy_tx_queue) { 640 .queue_id = cpu_to_be32(queue_index), 641 }; 642 643 err = gve_adminq_issue_cmd(priv, &cmd); 644 if (err) 645 return err; 646 647 return 0; 648} 649 650int gve_adminq_destroy_tx_queues(struct gve_priv *priv, u32 start_id, u32 num_queues) 651{ 652 int err; 653 int i; 654 655 for (i = start_id; i < start_id + num_queues; i++) { 656 err = gve_adminq_destroy_tx_queue(priv, i); 657 if (err) 658 return err; 659 } 660 661 return gve_adminq_kick_and_wait(priv); 662} 663 664static int gve_adminq_destroy_rx_queue(struct gve_priv *priv, u32 queue_index) 665{ 666 union gve_adminq_command cmd; 667 int err; 668 669 memset(&cmd, 0, sizeof(cmd)); 670 cmd.opcode = cpu_to_be32(GVE_ADMINQ_DESTROY_RX_QUEUE); 671 cmd.destroy_rx_queue = (struct gve_adminq_destroy_rx_queue) { 672 .queue_id = cpu_to_be32(queue_index), 673 }; 674 675 err = gve_adminq_issue_cmd(priv, &cmd); 676 if (err) 677 return err; 678 679 return 0; 680} 681 682int gve_adminq_destroy_rx_queues(struct gve_priv *priv, u32 num_queues) 683{ 684 int err; 685 int i; 686 687 for (i = 0; i < num_queues; i++) { 688 err = gve_adminq_destroy_rx_queue(priv, i); 689 if (err) 690 return err; 691 } 692 693 return gve_adminq_kick_and_wait(priv); 694} 695 696static int gve_set_desc_cnt(struct gve_priv *priv, 697 struct gve_device_descriptor *descriptor) 698{ 699 priv->tx_desc_cnt = be16_to_cpu(descriptor->tx_queue_entries); 700 if (priv->tx_desc_cnt * sizeof(priv->tx->desc[0]) < PAGE_SIZE) { 701 dev_err(&priv->pdev->dev, "Tx desc count %d too low\n", 702 priv->tx_desc_cnt); 703 return -EINVAL; 704 } 705 priv->rx_desc_cnt = be16_to_cpu(descriptor->rx_queue_entries); 706 if (priv->rx_desc_cnt * sizeof(priv->rx->desc.desc_ring[0]) 707 < PAGE_SIZE) { 708 dev_err(&priv->pdev->dev, "Rx desc count %d too low\n", 709 priv->rx_desc_cnt); 710 return -EINVAL; 711 } 712 return 0; 713} 714 715static int 716gve_set_desc_cnt_dqo(struct gve_priv *priv, 717 const struct gve_device_descriptor *descriptor, 718 const struct gve_device_option_dqo_rda *dev_op_dqo_rda) 719{ 720 priv->tx_desc_cnt = be16_to_cpu(descriptor->tx_queue_entries); 721 priv->rx_desc_cnt = be16_to_cpu(descriptor->rx_queue_entries); 722 723 if (priv->queue_format == GVE_DQO_QPL_FORMAT) 724 return 0; 725 726 priv->options_dqo_rda.tx_comp_ring_entries = 727 be16_to_cpu(dev_op_dqo_rda->tx_comp_ring_entries); 728 priv->options_dqo_rda.rx_buff_ring_entries = 729 be16_to_cpu(dev_op_dqo_rda->rx_buff_ring_entries); 730 731 return 0; 732} 733 734static void gve_enable_supported_features(struct gve_priv *priv, 735 u32 supported_features_mask, 736 const struct gve_device_option_jumbo_frames 737 *dev_op_jumbo_frames, 738 const struct gve_device_option_dqo_qpl 739 *dev_op_dqo_qpl) 740{ 741 /* Before control reaches this point, the page-size-capped max MTU from 742 * the gve_device_descriptor field has already been stored in 743 * priv->dev->max_mtu. We overwrite it with the true max MTU below. 744 */ 745 if (dev_op_jumbo_frames && 746 (supported_features_mask & GVE_SUP_JUMBO_FRAMES_MASK)) { 747 dev_info(&priv->pdev->dev, 748 "JUMBO FRAMES device option enabled.\n"); 749 priv->dev->max_mtu = be16_to_cpu(dev_op_jumbo_frames->max_mtu); 750 } 751 752 /* Override pages for qpl for DQO-QPL */ 753 if (dev_op_dqo_qpl) { 754 priv->tx_pages_per_qpl = 755 be16_to_cpu(dev_op_dqo_qpl->tx_pages_per_qpl); 756 priv->rx_pages_per_qpl = 757 be16_to_cpu(dev_op_dqo_qpl->rx_pages_per_qpl); 758 if (priv->tx_pages_per_qpl == 0) 759 priv->tx_pages_per_qpl = DQO_QPL_DEFAULT_TX_PAGES; 760 if (priv->rx_pages_per_qpl == 0) 761 priv->rx_pages_per_qpl = DQO_QPL_DEFAULT_RX_PAGES; 762 } 763} 764 765int gve_adminq_describe_device(struct gve_priv *priv) 766{ 767 struct gve_device_option_jumbo_frames *dev_op_jumbo_frames = NULL; 768 struct gve_device_option_gqi_rda *dev_op_gqi_rda = NULL; 769 struct gve_device_option_gqi_qpl *dev_op_gqi_qpl = NULL; 770 struct gve_device_option_dqo_rda *dev_op_dqo_rda = NULL; 771 struct gve_device_option_dqo_qpl *dev_op_dqo_qpl = NULL; 772 struct gve_device_descriptor *descriptor; 773 u32 supported_features_mask = 0; 774 union gve_adminq_command cmd; 775 dma_addr_t descriptor_bus; 776 int err = 0; 777 u8 *mac; 778 u16 mtu; 779 780 memset(&cmd, 0, sizeof(cmd)); 781 descriptor = dma_alloc_coherent(&priv->pdev->dev, PAGE_SIZE, 782 &descriptor_bus, GFP_KERNEL); 783 if (!descriptor) 784 return -ENOMEM; 785 cmd.opcode = cpu_to_be32(GVE_ADMINQ_DESCRIBE_DEVICE); 786 cmd.describe_device.device_descriptor_addr = 787 cpu_to_be64(descriptor_bus); 788 cmd.describe_device.device_descriptor_version = 789 cpu_to_be32(GVE_ADMINQ_DEVICE_DESCRIPTOR_VERSION); 790 cmd.describe_device.available_length = cpu_to_be32(PAGE_SIZE); 791 792 err = gve_adminq_execute_cmd(priv, &cmd); 793 if (err) 794 goto free_device_descriptor; 795 796 err = gve_process_device_options(priv, descriptor, &dev_op_gqi_rda, 797 &dev_op_gqi_qpl, &dev_op_dqo_rda, 798 &dev_op_jumbo_frames, 799 &dev_op_dqo_qpl); 800 if (err) 801 goto free_device_descriptor; 802 803 /* If the GQI_RAW_ADDRESSING option is not enabled and the queue format 804 * is not set to GqiRda, choose the queue format in a priority order: 805 * DqoRda, DqoQpl, GqiRda, GqiQpl. Use GqiQpl as default. 806 */ 807 if (dev_op_dqo_rda) { 808 priv->queue_format = GVE_DQO_RDA_FORMAT; 809 dev_info(&priv->pdev->dev, 810 "Driver is running with DQO RDA queue format.\n"); 811 supported_features_mask = 812 be32_to_cpu(dev_op_dqo_rda->supported_features_mask); 813 } else if (dev_op_dqo_qpl) { 814 priv->queue_format = GVE_DQO_QPL_FORMAT; 815 supported_features_mask = 816 be32_to_cpu(dev_op_dqo_qpl->supported_features_mask); 817 } else if (dev_op_gqi_rda) { 818 priv->queue_format = GVE_GQI_RDA_FORMAT; 819 dev_info(&priv->pdev->dev, 820 "Driver is running with GQI RDA queue format.\n"); 821 supported_features_mask = 822 be32_to_cpu(dev_op_gqi_rda->supported_features_mask); 823 } else if (priv->queue_format == GVE_GQI_RDA_FORMAT) { 824 dev_info(&priv->pdev->dev, 825 "Driver is running with GQI RDA queue format.\n"); 826 } else { 827 priv->queue_format = GVE_GQI_QPL_FORMAT; 828 if (dev_op_gqi_qpl) 829 supported_features_mask = 830 be32_to_cpu(dev_op_gqi_qpl->supported_features_mask); 831 dev_info(&priv->pdev->dev, 832 "Driver is running with GQI QPL queue format.\n"); 833 } 834 if (gve_is_gqi(priv)) { 835 err = gve_set_desc_cnt(priv, descriptor); 836 } else { 837 /* DQO supports LRO. */ 838 priv->dev->hw_features |= NETIF_F_LRO; 839 err = gve_set_desc_cnt_dqo(priv, descriptor, dev_op_dqo_rda); 840 } 841 if (err) 842 goto free_device_descriptor; 843 844 priv->max_registered_pages = 845 be64_to_cpu(descriptor->max_registered_pages); 846 mtu = be16_to_cpu(descriptor->mtu); 847 if (mtu < ETH_MIN_MTU) { 848 dev_err(&priv->pdev->dev, "MTU %d below minimum MTU\n", mtu); 849 err = -EINVAL; 850 goto free_device_descriptor; 851 } 852 priv->dev->max_mtu = mtu; 853 priv->num_event_counters = be16_to_cpu(descriptor->counters); 854 eth_hw_addr_set(priv->dev, descriptor->mac); 855 mac = descriptor->mac; 856 dev_info(&priv->pdev->dev, "MAC addr: %pM\n", mac); 857 priv->tx_pages_per_qpl = be16_to_cpu(descriptor->tx_pages_per_qpl); 858 priv->rx_data_slot_cnt = be16_to_cpu(descriptor->rx_pages_per_qpl); 859 860 if (gve_is_gqi(priv) && priv->rx_data_slot_cnt < priv->rx_desc_cnt) { 861 dev_err(&priv->pdev->dev, "rx_data_slot_cnt cannot be smaller than rx_desc_cnt, setting rx_desc_cnt down to %d.\n", 862 priv->rx_data_slot_cnt); 863 priv->rx_desc_cnt = priv->rx_data_slot_cnt; 864 } 865 priv->default_num_queues = be16_to_cpu(descriptor->default_num_queues); 866 867 gve_enable_supported_features(priv, supported_features_mask, 868 dev_op_jumbo_frames, dev_op_dqo_qpl); 869 870free_device_descriptor: 871 dma_free_coherent(&priv->pdev->dev, PAGE_SIZE, descriptor, 872 descriptor_bus); 873 return err; 874} 875 876int gve_adminq_register_page_list(struct gve_priv *priv, 877 struct gve_queue_page_list *qpl) 878{ 879 struct device *hdev = &priv->pdev->dev; 880 u32 num_entries = qpl->num_entries; 881 u32 size = num_entries * sizeof(qpl->page_buses[0]); 882 union gve_adminq_command cmd; 883 dma_addr_t page_list_bus; 884 __be64 *page_list; 885 int err; 886 int i; 887 888 memset(&cmd, 0, sizeof(cmd)); 889 page_list = dma_alloc_coherent(hdev, size, &page_list_bus, GFP_KERNEL); 890 if (!page_list) 891 return -ENOMEM; 892 893 for (i = 0; i < num_entries; i++) 894 page_list[i] = cpu_to_be64(qpl->page_buses[i]); 895 896 cmd.opcode = cpu_to_be32(GVE_ADMINQ_REGISTER_PAGE_LIST); 897 cmd.reg_page_list = (struct gve_adminq_register_page_list) { 898 .page_list_id = cpu_to_be32(qpl->id), 899 .num_pages = cpu_to_be32(num_entries), 900 .page_address_list_addr = cpu_to_be64(page_list_bus), 901 }; 902 903 err = gve_adminq_execute_cmd(priv, &cmd); 904 dma_free_coherent(hdev, size, page_list, page_list_bus); 905 return err; 906} 907 908int gve_adminq_unregister_page_list(struct gve_priv *priv, u32 page_list_id) 909{ 910 union gve_adminq_command cmd; 911 912 memset(&cmd, 0, sizeof(cmd)); 913 cmd.opcode = cpu_to_be32(GVE_ADMINQ_UNREGISTER_PAGE_LIST); 914 cmd.unreg_page_list = (struct gve_adminq_unregister_page_list) { 915 .page_list_id = cpu_to_be32(page_list_id), 916 }; 917 918 return gve_adminq_execute_cmd(priv, &cmd); 919} 920 921int gve_adminq_set_mtu(struct gve_priv *priv, u64 mtu) 922{ 923 union gve_adminq_command cmd; 924 925 memset(&cmd, 0, sizeof(cmd)); 926 cmd.opcode = cpu_to_be32(GVE_ADMINQ_SET_DRIVER_PARAMETER); 927 cmd.set_driver_param = (struct gve_adminq_set_driver_parameter) { 928 .parameter_type = cpu_to_be32(GVE_SET_PARAM_MTU), 929 .parameter_value = cpu_to_be64(mtu), 930 }; 931 932 return gve_adminq_execute_cmd(priv, &cmd); 933} 934 935int gve_adminq_report_stats(struct gve_priv *priv, u64 stats_report_len, 936 dma_addr_t stats_report_addr, u64 interval) 937{ 938 union gve_adminq_command cmd; 939 940 memset(&cmd, 0, sizeof(cmd)); 941 cmd.opcode = cpu_to_be32(GVE_ADMINQ_REPORT_STATS); 942 cmd.report_stats = (struct gve_adminq_report_stats) { 943 .stats_report_len = cpu_to_be64(stats_report_len), 944 .stats_report_addr = cpu_to_be64(stats_report_addr), 945 .interval = cpu_to_be64(interval), 946 }; 947 948 return gve_adminq_execute_cmd(priv, &cmd); 949} 950 951int gve_adminq_verify_driver_compatibility(struct gve_priv *priv, 952 u64 driver_info_len, 953 dma_addr_t driver_info_addr) 954{ 955 union gve_adminq_command cmd; 956 957 memset(&cmd, 0, sizeof(cmd)); 958 cmd.opcode = cpu_to_be32(GVE_ADMINQ_VERIFY_DRIVER_COMPATIBILITY); 959 cmd.verify_driver_compatibility = (struct gve_adminq_verify_driver_compatibility) { 960 .driver_info_len = cpu_to_be64(driver_info_len), 961 .driver_info_addr = cpu_to_be64(driver_info_addr), 962 }; 963 964 return gve_adminq_execute_cmd(priv, &cmd); 965} 966 967int gve_adminq_report_link_speed(struct gve_priv *priv) 968{ 969 union gve_adminq_command gvnic_cmd; 970 dma_addr_t link_speed_region_bus; 971 __be64 *link_speed_region; 972 int err; 973 974 link_speed_region = 975 dma_alloc_coherent(&priv->pdev->dev, sizeof(*link_speed_region), 976 &link_speed_region_bus, GFP_KERNEL); 977 978 if (!link_speed_region) 979 return -ENOMEM; 980 981 memset(&gvnic_cmd, 0, sizeof(gvnic_cmd)); 982 gvnic_cmd.opcode = cpu_to_be32(GVE_ADMINQ_REPORT_LINK_SPEED); 983 gvnic_cmd.report_link_speed.link_speed_address = 984 cpu_to_be64(link_speed_region_bus); 985 986 err = gve_adminq_execute_cmd(priv, &gvnic_cmd); 987 988 priv->link_speed = be64_to_cpu(*link_speed_region); 989 dma_free_coherent(&priv->pdev->dev, sizeof(*link_speed_region), link_speed_region, 990 link_speed_region_bus); 991 return err; 992} 993 994int gve_adminq_get_ptype_map_dqo(struct gve_priv *priv, 995 struct gve_ptype_lut *ptype_lut) 996{ 997 struct gve_ptype_map *ptype_map; 998 union gve_adminq_command cmd; 999 dma_addr_t ptype_map_bus; 1000 int err = 0; 1001 int i; 1002 1003 memset(&cmd, 0, sizeof(cmd)); 1004 ptype_map = dma_alloc_coherent(&priv->pdev->dev, sizeof(*ptype_map), 1005 &ptype_map_bus, GFP_KERNEL); 1006 if (!ptype_map) 1007 return -ENOMEM; 1008 1009 cmd.opcode = cpu_to_be32(GVE_ADMINQ_GET_PTYPE_MAP); 1010 cmd.get_ptype_map = (struct gve_adminq_get_ptype_map) { 1011 .ptype_map_len = cpu_to_be64(sizeof(*ptype_map)), 1012 .ptype_map_addr = cpu_to_be64(ptype_map_bus), 1013 }; 1014 1015 err = gve_adminq_execute_cmd(priv, &cmd); 1016 if (err) 1017 goto err; 1018 1019 /* Populate ptype_lut. */ 1020 for (i = 0; i < GVE_NUM_PTYPES; i++) { 1021 ptype_lut->ptypes[i].l3_type = 1022 ptype_map->ptypes[i].l3_type; 1023 ptype_lut->ptypes[i].l4_type = 1024 ptype_map->ptypes[i].l4_type; 1025 } 1026err: 1027 dma_free_coherent(&priv->pdev->dev, sizeof(*ptype_map), ptype_map, 1028 ptype_map_bus); 1029 return err; 1030} 1031