1/* 2 * Back-end of the driver for virtual network devices. This portion of the 3 * driver exports a 'unified' network-device interface that can be accessed 4 * by any operating system that implements a compatible front end. A 5 * reference front-end implementation can be found in: 6 * drivers/net/xen-netfront.c 7 * 8 * Copyright (c) 2002-2005, K A Fraser 9 * 10 * This program is free software; you can redistribute it and/or 11 * modify it under the terms of the GNU General Public License version 2 12 * as published by the Free Software Foundation; or, when distributed 13 * separately from the Linux kernel or incorporated into other 14 * software packages, subject to the following license: 15 * 16 * Permission is hereby granted, free of charge, to any person obtaining a copy 17 * of this source file (the "Software"), to deal in the Software without 18 * restriction, including without limitation the rights to use, copy, modify, 19 * merge, publish, distribute, sublicense, and/or sell copies of the Software, 20 * and to permit persons to whom the Software is furnished to do so, subject to 21 * the following conditions: 22 * 23 * The above copyright notice and this permission notice shall be included in 24 * all copies or substantial portions of the Software. 25 * 26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 27 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 28 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 29 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 30 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 31 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 32 * IN THE SOFTWARE. 33 */ 34 35#include "common.h" 36 37#include <linux/kthread.h> 38#include <linux/if_vlan.h> 39#include <linux/udp.h> 40#include <linux/highmem.h> 41 42#include <net/tcp.h> 43 44#include <xen/xen.h> 45#include <xen/events.h> 46#include <xen/interface/memory.h> 47#include <xen/page.h> 48 49#include <asm/xen/hypercall.h> 50 51/* Provide an option to disable split event channels at load time as 52 * event channels are limited resource. Split event channels are 53 * enabled by default. 54 */ 55bool separate_tx_rx_irq = true; 56module_param(separate_tx_rx_irq, bool, 0644); 57 58/* The time that packets can stay on the guest Rx internal queue 59 * before they are dropped. 60 */ 61unsigned int rx_drain_timeout_msecs = 10000; 62module_param(rx_drain_timeout_msecs, uint, 0444); 63 64/* The length of time before the frontend is considered unresponsive 65 * because it isn't providing Rx slots. 66 */ 67unsigned int rx_stall_timeout_msecs = 60000; 68module_param(rx_stall_timeout_msecs, uint, 0444); 69 70#define MAX_QUEUES_DEFAULT 8 71unsigned int xenvif_max_queues; 72module_param_named(max_queues, xenvif_max_queues, uint, 0644); 73MODULE_PARM_DESC(max_queues, 74 "Maximum number of queues per virtual interface"); 75 76/* 77 * This is the maximum slots a skb can have. If a guest sends a skb 78 * which exceeds this limit it is considered malicious. 79 */ 80#define FATAL_SKB_SLOTS_DEFAULT 20 81static unsigned int fatal_skb_slots = FATAL_SKB_SLOTS_DEFAULT; 82module_param(fatal_skb_slots, uint, 0444); 83 84/* The amount to copy out of the first guest Tx slot into the skb's 85 * linear area. If the first slot has more data, it will be mapped 86 * and put into the first frag. 87 * 88 * This is sized to avoid pulling headers from the frags for most 89 * TCP/IP packets. 90 */ 91#define XEN_NETBACK_TX_COPY_LEN 128 92 93/* This is the maximum number of flows in the hash cache. */ 94#define XENVIF_HASH_CACHE_SIZE_DEFAULT 64 95unsigned int xenvif_hash_cache_size = XENVIF_HASH_CACHE_SIZE_DEFAULT; 96module_param_named(hash_cache_size, xenvif_hash_cache_size, uint, 0644); 97MODULE_PARM_DESC(hash_cache_size, "Number of flows in the hash cache"); 98 99/* The module parameter tells that we have to put data 100 * for xen-netfront with the XDP_PACKET_HEADROOM offset 101 * needed for XDP processing 102 */ 103bool provides_xdp_headroom = true; 104module_param(provides_xdp_headroom, bool, 0644); 105 106static void xenvif_idx_release(struct xenvif_queue *queue, u16 pending_idx, 107 s8 status); 108 109static void make_tx_response(struct xenvif_queue *queue, 110 const struct xen_netif_tx_request *txp, 111 unsigned int extra_count, 112 s8 status); 113 114static void xenvif_idx_unmap(struct xenvif_queue *queue, u16 pending_idx); 115 116static inline int tx_work_todo(struct xenvif_queue *queue); 117 118static inline unsigned long idx_to_pfn(struct xenvif_queue *queue, 119 u16 idx) 120{ 121 return page_to_pfn(queue->mmap_pages[idx]); 122} 123 124static inline unsigned long idx_to_kaddr(struct xenvif_queue *queue, 125 u16 idx) 126{ 127 return (unsigned long)pfn_to_kaddr(idx_to_pfn(queue, idx)); 128} 129 130#define callback_param(vif, pending_idx) \ 131 (vif->pending_tx_info[pending_idx].callback_struct) 132 133/* Find the containing VIF's structure from a pointer in pending_tx_info array 134 */ 135static inline struct xenvif_queue *ubuf_to_queue(const struct ubuf_info *ubuf) 136{ 137 u16 pending_idx = ubuf->desc; 138 struct pending_tx_info *temp = 139 container_of(ubuf, struct pending_tx_info, callback_struct); 140 return container_of(temp - pending_idx, 141 struct xenvif_queue, 142 pending_tx_info[0]); 143} 144 145static u16 frag_get_pending_idx(skb_frag_t *frag) 146{ 147 return (u16)skb_frag_off(frag); 148} 149 150static void frag_set_pending_idx(skb_frag_t *frag, u16 pending_idx) 151{ 152 skb_frag_off_set(frag, pending_idx); 153} 154 155static inline pending_ring_idx_t pending_index(unsigned i) 156{ 157 return i & (MAX_PENDING_REQS-1); 158} 159 160void xenvif_kick_thread(struct xenvif_queue *queue) 161{ 162 wake_up(&queue->wq); 163} 164 165void xenvif_napi_schedule_or_enable_events(struct xenvif_queue *queue) 166{ 167 int more_to_do; 168 169 RING_FINAL_CHECK_FOR_REQUESTS(&queue->tx, more_to_do); 170 171 if (more_to_do) 172 napi_schedule(&queue->napi); 173 else if (atomic_fetch_andnot(NETBK_TX_EOI | NETBK_COMMON_EOI, 174 &queue->eoi_pending) & 175 (NETBK_TX_EOI | NETBK_COMMON_EOI)) 176 xen_irq_lateeoi(queue->tx_irq, 0); 177} 178 179static void tx_add_credit(struct xenvif_queue *queue) 180{ 181 unsigned long max_burst, max_credit; 182 183 /* 184 * Allow a burst big enough to transmit a jumbo packet of up to 128kB. 185 * Otherwise the interface can seize up due to insufficient credit. 186 */ 187 max_burst = max(131072UL, queue->credit_bytes); 188 189 /* Take care that adding a new chunk of credit doesn't wrap to zero. */ 190 max_credit = queue->remaining_credit + queue->credit_bytes; 191 if (max_credit < queue->remaining_credit) 192 max_credit = ULONG_MAX; /* wrapped: clamp to ULONG_MAX */ 193 194 queue->remaining_credit = min(max_credit, max_burst); 195 queue->rate_limited = false; 196} 197 198void xenvif_tx_credit_callback(struct timer_list *t) 199{ 200 struct xenvif_queue *queue = from_timer(queue, t, credit_timeout); 201 tx_add_credit(queue); 202 xenvif_napi_schedule_or_enable_events(queue); 203} 204 205static void xenvif_tx_err(struct xenvif_queue *queue, 206 struct xen_netif_tx_request *txp, 207 unsigned int extra_count, RING_IDX end) 208{ 209 RING_IDX cons = queue->tx.req_cons; 210 211 do { 212 make_tx_response(queue, txp, extra_count, XEN_NETIF_RSP_ERROR); 213 if (cons == end) 214 break; 215 RING_COPY_REQUEST(&queue->tx, cons++, txp); 216 extra_count = 0; /* only the first frag can have extras */ 217 } while (1); 218 queue->tx.req_cons = cons; 219} 220 221static void xenvif_fatal_tx_err(struct xenvif *vif) 222{ 223 netdev_err(vif->dev, "fatal error; disabling device\n"); 224 vif->disabled = true; 225 /* Disable the vif from queue 0's kthread */ 226 if (vif->num_queues) 227 xenvif_kick_thread(&vif->queues[0]); 228} 229 230static int xenvif_count_requests(struct xenvif_queue *queue, 231 struct xen_netif_tx_request *first, 232 unsigned int extra_count, 233 struct xen_netif_tx_request *txp, 234 int work_to_do) 235{ 236 RING_IDX cons = queue->tx.req_cons; 237 int slots = 0; 238 int drop_err = 0; 239 int more_data; 240 241 if (!(first->flags & XEN_NETTXF_more_data)) 242 return 0; 243 244 do { 245 struct xen_netif_tx_request dropped_tx = { 0 }; 246 247 if (slots >= work_to_do) { 248 netdev_err(queue->vif->dev, 249 "Asked for %d slots but exceeds this limit\n", 250 work_to_do); 251 xenvif_fatal_tx_err(queue->vif); 252 return -ENODATA; 253 } 254 255 /* This guest is really using too many slots and 256 * considered malicious. 257 */ 258 if (unlikely(slots >= fatal_skb_slots)) { 259 netdev_err(queue->vif->dev, 260 "Malicious frontend using %d slots, threshold %u\n", 261 slots, fatal_skb_slots); 262 xenvif_fatal_tx_err(queue->vif); 263 return -E2BIG; 264 } 265 266 /* Xen network protocol had implicit dependency on 267 * MAX_SKB_FRAGS. XEN_NETBK_LEGACY_SLOTS_MAX is set to 268 * the historical MAX_SKB_FRAGS value 18 to honor the 269 * same behavior as before. Any packet using more than 270 * 18 slots but less than fatal_skb_slots slots is 271 * dropped 272 */ 273 if (!drop_err && slots >= XEN_NETBK_LEGACY_SLOTS_MAX) { 274 if (net_ratelimit()) 275 netdev_dbg(queue->vif->dev, 276 "Too many slots (%d) exceeding limit (%d), dropping packet\n", 277 slots, XEN_NETBK_LEGACY_SLOTS_MAX); 278 drop_err = -E2BIG; 279 } 280 281 if (drop_err) 282 txp = &dropped_tx; 283 284 RING_COPY_REQUEST(&queue->tx, cons + slots, txp); 285 286 /* If the guest submitted a frame >= 64 KiB then 287 * first->size overflowed and following slots will 288 * appear to be larger than the frame. 289 * 290 * This cannot be fatal error as there are buggy 291 * frontends that do this. 292 * 293 * Consume all slots and drop the packet. 294 */ 295 if (!drop_err && txp->size > first->size) { 296 if (net_ratelimit()) 297 netdev_dbg(queue->vif->dev, 298 "Invalid tx request, slot size %u > remaining size %u\n", 299 txp->size, first->size); 300 drop_err = -EIO; 301 } 302 303 first->size -= txp->size; 304 slots++; 305 306 if (unlikely((txp->offset + txp->size) > XEN_PAGE_SIZE)) { 307 netdev_err(queue->vif->dev, "Cross page boundary, txp->offset: %u, size: %u\n", 308 txp->offset, txp->size); 309 xenvif_fatal_tx_err(queue->vif); 310 return -EINVAL; 311 } 312 313 more_data = txp->flags & XEN_NETTXF_more_data; 314 315 if (!drop_err) 316 txp++; 317 318 } while (more_data); 319 320 if (drop_err) { 321 xenvif_tx_err(queue, first, extra_count, cons + slots); 322 return drop_err; 323 } 324 325 return slots; 326} 327 328 329struct xenvif_tx_cb { 330 u16 copy_pending_idx[XEN_NETBK_LEGACY_SLOTS_MAX + 1]; 331 u8 copy_count; 332 u32 split_mask; 333}; 334 335#define XENVIF_TX_CB(skb) ((struct xenvif_tx_cb *)(skb)->cb) 336#define copy_pending_idx(skb, i) (XENVIF_TX_CB(skb)->copy_pending_idx[i]) 337#define copy_count(skb) (XENVIF_TX_CB(skb)->copy_count) 338 339static inline void xenvif_tx_create_map_op(struct xenvif_queue *queue, 340 u16 pending_idx, 341 struct xen_netif_tx_request *txp, 342 unsigned int extra_count, 343 struct gnttab_map_grant_ref *mop) 344{ 345 queue->pages_to_map[mop-queue->tx_map_ops] = queue->mmap_pages[pending_idx]; 346 gnttab_set_map_op(mop, idx_to_kaddr(queue, pending_idx), 347 GNTMAP_host_map | GNTMAP_readonly, 348 txp->gref, queue->vif->domid); 349 350 memcpy(&queue->pending_tx_info[pending_idx].req, txp, 351 sizeof(*txp)); 352 queue->pending_tx_info[pending_idx].extra_count = extra_count; 353} 354 355static inline struct sk_buff *xenvif_alloc_skb(unsigned int size) 356{ 357 struct sk_buff *skb = 358 alloc_skb(size + NET_SKB_PAD + NET_IP_ALIGN, 359 GFP_ATOMIC | __GFP_NOWARN); 360 361 BUILD_BUG_ON(sizeof(*XENVIF_TX_CB(skb)) > sizeof(skb->cb)); 362 if (unlikely(skb == NULL)) 363 return NULL; 364 365 /* Packets passed to netif_rx() must have some headroom. */ 366 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN); 367 368 /* Initialize it here to avoid later surprises */ 369 skb_shinfo(skb)->destructor_arg = NULL; 370 371 return skb; 372} 373 374static void xenvif_get_requests(struct xenvif_queue *queue, 375 struct sk_buff *skb, 376 struct xen_netif_tx_request *first, 377 struct xen_netif_tx_request *txfrags, 378 unsigned *copy_ops, 379 unsigned *map_ops, 380 unsigned int frag_overflow, 381 struct sk_buff *nskb, 382 unsigned int extra_count, 383 unsigned int data_len) 384{ 385 struct skb_shared_info *shinfo = skb_shinfo(skb); 386 skb_frag_t *frags = shinfo->frags; 387 u16 pending_idx; 388 pending_ring_idx_t index; 389 unsigned int nr_slots; 390 struct gnttab_copy *cop = queue->tx_copy_ops + *copy_ops; 391 struct gnttab_map_grant_ref *gop = queue->tx_map_ops + *map_ops; 392 struct xen_netif_tx_request *txp = first; 393 394 nr_slots = shinfo->nr_frags + frag_overflow + 1; 395 396 copy_count(skb) = 0; 397 XENVIF_TX_CB(skb)->split_mask = 0; 398 399 /* Create copy ops for exactly data_len bytes into the skb head. */ 400 __skb_put(skb, data_len); 401 while (data_len > 0) { 402 int amount = data_len > txp->size ? txp->size : data_len; 403 bool split = false; 404 405 cop->source.u.ref = txp->gref; 406 cop->source.domid = queue->vif->domid; 407 cop->source.offset = txp->offset; 408 409 cop->dest.domid = DOMID_SELF; 410 cop->dest.offset = (offset_in_page(skb->data + 411 skb_headlen(skb) - 412 data_len)) & ~XEN_PAGE_MASK; 413 cop->dest.u.gmfn = virt_to_gfn(skb->data + skb_headlen(skb) 414 - data_len); 415 416 /* Don't cross local page boundary! */ 417 if (cop->dest.offset + amount > XEN_PAGE_SIZE) { 418 amount = XEN_PAGE_SIZE - cop->dest.offset; 419 XENVIF_TX_CB(skb)->split_mask |= 1U << copy_count(skb); 420 split = true; 421 } 422 423 cop->len = amount; 424 cop->flags = GNTCOPY_source_gref; 425 426 index = pending_index(queue->pending_cons); 427 pending_idx = queue->pending_ring[index]; 428 callback_param(queue, pending_idx).ctx = NULL; 429 copy_pending_idx(skb, copy_count(skb)) = pending_idx; 430 if (!split) 431 copy_count(skb)++; 432 433 cop++; 434 data_len -= amount; 435 436 if (amount == txp->size) { 437 /* The copy op covered the full tx_request */ 438 439 memcpy(&queue->pending_tx_info[pending_idx].req, 440 txp, sizeof(*txp)); 441 queue->pending_tx_info[pending_idx].extra_count = 442 (txp == first) ? extra_count : 0; 443 444 if (txp == first) 445 txp = txfrags; 446 else 447 txp++; 448 queue->pending_cons++; 449 nr_slots--; 450 } else { 451 /* The copy op partially covered the tx_request. 452 * The remainder will be mapped or copied in the next 453 * iteration. 454 */ 455 txp->offset += amount; 456 txp->size -= amount; 457 } 458 } 459 460 for (shinfo->nr_frags = 0; nr_slots > 0 && shinfo->nr_frags < MAX_SKB_FRAGS; 461 nr_slots--) { 462 if (unlikely(!txp->size)) { 463 make_tx_response(queue, txp, 0, XEN_NETIF_RSP_OKAY); 464 ++txp; 465 continue; 466 } 467 468 index = pending_index(queue->pending_cons++); 469 pending_idx = queue->pending_ring[index]; 470 xenvif_tx_create_map_op(queue, pending_idx, txp, 471 txp == first ? extra_count : 0, gop); 472 frag_set_pending_idx(&frags[shinfo->nr_frags], pending_idx); 473 ++shinfo->nr_frags; 474 ++gop; 475 476 if (txp == first) 477 txp = txfrags; 478 else 479 txp++; 480 } 481 482 if (nr_slots > 0) { 483 484 shinfo = skb_shinfo(nskb); 485 frags = shinfo->frags; 486 487 for (shinfo->nr_frags = 0; shinfo->nr_frags < nr_slots; ++txp) { 488 if (unlikely(!txp->size)) { 489 make_tx_response(queue, txp, 0, 490 XEN_NETIF_RSP_OKAY); 491 continue; 492 } 493 494 index = pending_index(queue->pending_cons++); 495 pending_idx = queue->pending_ring[index]; 496 xenvif_tx_create_map_op(queue, pending_idx, txp, 0, 497 gop); 498 frag_set_pending_idx(&frags[shinfo->nr_frags], 499 pending_idx); 500 ++shinfo->nr_frags; 501 ++gop; 502 } 503 504 if (shinfo->nr_frags) { 505 skb_shinfo(skb)->frag_list = nskb; 506 nskb = NULL; 507 } 508 } 509 510 if (nskb) { 511 /* A frag_list skb was allocated but it is no longer needed 512 * because enough slots were converted to copy ops above or some 513 * were empty. 514 */ 515 kfree_skb(nskb); 516 } 517 518 (*copy_ops) = cop - queue->tx_copy_ops; 519 (*map_ops) = gop - queue->tx_map_ops; 520} 521 522static inline void xenvif_grant_handle_set(struct xenvif_queue *queue, 523 u16 pending_idx, 524 grant_handle_t handle) 525{ 526 if (unlikely(queue->grant_tx_handle[pending_idx] != 527 NETBACK_INVALID_HANDLE)) { 528 netdev_err(queue->vif->dev, 529 "Trying to overwrite active handle! pending_idx: 0x%x\n", 530 pending_idx); 531 BUG(); 532 } 533 queue->grant_tx_handle[pending_idx] = handle; 534} 535 536static inline void xenvif_grant_handle_reset(struct xenvif_queue *queue, 537 u16 pending_idx) 538{ 539 if (unlikely(queue->grant_tx_handle[pending_idx] == 540 NETBACK_INVALID_HANDLE)) { 541 netdev_err(queue->vif->dev, 542 "Trying to unmap invalid handle! pending_idx: 0x%x\n", 543 pending_idx); 544 BUG(); 545 } 546 queue->grant_tx_handle[pending_idx] = NETBACK_INVALID_HANDLE; 547} 548 549static int xenvif_tx_check_gop(struct xenvif_queue *queue, 550 struct sk_buff *skb, 551 struct gnttab_map_grant_ref **gopp_map, 552 struct gnttab_copy **gopp_copy) 553{ 554 struct gnttab_map_grant_ref *gop_map = *gopp_map; 555 u16 pending_idx; 556 /* This always points to the shinfo of the skb being checked, which 557 * could be either the first or the one on the frag_list 558 */ 559 struct skb_shared_info *shinfo = skb_shinfo(skb); 560 /* If this is non-NULL, we are currently checking the frag_list skb, and 561 * this points to the shinfo of the first one 562 */ 563 struct skb_shared_info *first_shinfo = NULL; 564 int nr_frags = shinfo->nr_frags; 565 const bool sharedslot = nr_frags && 566 frag_get_pending_idx(&shinfo->frags[0]) == 567 copy_pending_idx(skb, copy_count(skb) - 1); 568 int i, err = 0; 569 570 for (i = 0; i < copy_count(skb); i++) { 571 int newerr; 572 573 /* Check status of header. */ 574 pending_idx = copy_pending_idx(skb, i); 575 576 newerr = (*gopp_copy)->status; 577 578 /* Split copies need to be handled together. */ 579 if (XENVIF_TX_CB(skb)->split_mask & (1U << i)) { 580 (*gopp_copy)++; 581 if (!newerr) 582 newerr = (*gopp_copy)->status; 583 } 584 if (likely(!newerr)) { 585 /* The first frag might still have this slot mapped */ 586 if (i < copy_count(skb) - 1 || !sharedslot) 587 xenvif_idx_release(queue, pending_idx, 588 XEN_NETIF_RSP_OKAY); 589 } else { 590 err = newerr; 591 if (net_ratelimit()) 592 netdev_dbg(queue->vif->dev, 593 "Grant copy of header failed! status: %d pending_idx: %u ref: %u\n", 594 (*gopp_copy)->status, 595 pending_idx, 596 (*gopp_copy)->source.u.ref); 597 /* The first frag might still have this slot mapped */ 598 if (i < copy_count(skb) - 1 || !sharedslot) 599 xenvif_idx_release(queue, pending_idx, 600 XEN_NETIF_RSP_ERROR); 601 } 602 (*gopp_copy)++; 603 } 604 605check_frags: 606 for (i = 0; i < nr_frags; i++, gop_map++) { 607 int j, newerr; 608 609 pending_idx = frag_get_pending_idx(&shinfo->frags[i]); 610 611 /* Check error status: if okay then remember grant handle. */ 612 newerr = gop_map->status; 613 614 if (likely(!newerr)) { 615 xenvif_grant_handle_set(queue, 616 pending_idx, 617 gop_map->handle); 618 /* Had a previous error? Invalidate this fragment. */ 619 if (unlikely(err)) { 620 xenvif_idx_unmap(queue, pending_idx); 621 /* If the mapping of the first frag was OK, but 622 * the header's copy failed, and they are 623 * sharing a slot, send an error 624 */ 625 if (i == 0 && !first_shinfo && sharedslot) 626 xenvif_idx_release(queue, pending_idx, 627 XEN_NETIF_RSP_ERROR); 628 else 629 xenvif_idx_release(queue, pending_idx, 630 XEN_NETIF_RSP_OKAY); 631 } 632 continue; 633 } 634 635 /* Error on this fragment: respond to client with an error. */ 636 if (net_ratelimit()) 637 netdev_dbg(queue->vif->dev, 638 "Grant map of %d. frag failed! status: %d pending_idx: %u ref: %u\n", 639 i, 640 gop_map->status, 641 pending_idx, 642 gop_map->ref); 643 644 xenvif_idx_release(queue, pending_idx, XEN_NETIF_RSP_ERROR); 645 646 /* Not the first error? Preceding frags already invalidated. */ 647 if (err) 648 continue; 649 650 /* Invalidate preceding fragments of this skb. */ 651 for (j = 0; j < i; j++) { 652 pending_idx = frag_get_pending_idx(&shinfo->frags[j]); 653 xenvif_idx_unmap(queue, pending_idx); 654 xenvif_idx_release(queue, pending_idx, 655 XEN_NETIF_RSP_OKAY); 656 } 657 658 /* And if we found the error while checking the frag_list, unmap 659 * the first skb's frags 660 */ 661 if (first_shinfo) { 662 for (j = 0; j < first_shinfo->nr_frags; j++) { 663 pending_idx = frag_get_pending_idx(&first_shinfo->frags[j]); 664 xenvif_idx_unmap(queue, pending_idx); 665 xenvif_idx_release(queue, pending_idx, 666 XEN_NETIF_RSP_OKAY); 667 } 668 } 669 670 /* Remember the error: invalidate all subsequent fragments. */ 671 err = newerr; 672 } 673 674 if (skb_has_frag_list(skb) && !first_shinfo) { 675 first_shinfo = skb_shinfo(skb); 676 shinfo = skb_shinfo(skb_shinfo(skb)->frag_list); 677 nr_frags = shinfo->nr_frags; 678 679 goto check_frags; 680 } 681 682 *gopp_map = gop_map; 683 return err; 684} 685 686static void xenvif_fill_frags(struct xenvif_queue *queue, struct sk_buff *skb) 687{ 688 struct skb_shared_info *shinfo = skb_shinfo(skb); 689 int nr_frags = shinfo->nr_frags; 690 int i; 691 u16 prev_pending_idx = INVALID_PENDING_IDX; 692 693 for (i = 0; i < nr_frags; i++) { 694 skb_frag_t *frag = shinfo->frags + i; 695 struct xen_netif_tx_request *txp; 696 struct page *page; 697 u16 pending_idx; 698 699 pending_idx = frag_get_pending_idx(frag); 700 701 /* If this is not the first frag, chain it to the previous*/ 702 if (prev_pending_idx == INVALID_PENDING_IDX) 703 skb_shinfo(skb)->destructor_arg = 704 &callback_param(queue, pending_idx); 705 else 706 callback_param(queue, prev_pending_idx).ctx = 707 &callback_param(queue, pending_idx); 708 709 callback_param(queue, pending_idx).ctx = NULL; 710 prev_pending_idx = pending_idx; 711 712 txp = &queue->pending_tx_info[pending_idx].req; 713 page = virt_to_page(idx_to_kaddr(queue, pending_idx)); 714 __skb_fill_page_desc(skb, i, page, txp->offset, txp->size); 715 skb->len += txp->size; 716 skb->data_len += txp->size; 717 skb->truesize += txp->size; 718 719 /* Take an extra reference to offset network stack's put_page */ 720 get_page(queue->mmap_pages[pending_idx]); 721 } 722} 723 724static int xenvif_get_extras(struct xenvif_queue *queue, 725 struct xen_netif_extra_info *extras, 726 unsigned int *extra_count, 727 int work_to_do) 728{ 729 struct xen_netif_extra_info extra; 730 RING_IDX cons = queue->tx.req_cons; 731 732 do { 733 if (unlikely(work_to_do-- <= 0)) { 734 netdev_err(queue->vif->dev, "Missing extra info\n"); 735 xenvif_fatal_tx_err(queue->vif); 736 return -EBADR; 737 } 738 739 RING_COPY_REQUEST(&queue->tx, cons, &extra); 740 741 queue->tx.req_cons = ++cons; 742 (*extra_count)++; 743 744 if (unlikely(!extra.type || 745 extra.type >= XEN_NETIF_EXTRA_TYPE_MAX)) { 746 netdev_err(queue->vif->dev, 747 "Invalid extra type: %d\n", extra.type); 748 xenvif_fatal_tx_err(queue->vif); 749 return -EINVAL; 750 } 751 752 memcpy(&extras[extra.type - 1], &extra, sizeof(extra)); 753 } while (extra.flags & XEN_NETIF_EXTRA_FLAG_MORE); 754 755 return work_to_do; 756} 757 758static int xenvif_set_skb_gso(struct xenvif *vif, 759 struct sk_buff *skb, 760 struct xen_netif_extra_info *gso) 761{ 762 if (!gso->u.gso.size) { 763 netdev_err(vif->dev, "GSO size must not be zero.\n"); 764 xenvif_fatal_tx_err(vif); 765 return -EINVAL; 766 } 767 768 switch (gso->u.gso.type) { 769 case XEN_NETIF_GSO_TYPE_TCPV4: 770 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4; 771 break; 772 case XEN_NETIF_GSO_TYPE_TCPV6: 773 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6; 774 break; 775 default: 776 netdev_err(vif->dev, "Bad GSO type %d.\n", gso->u.gso.type); 777 xenvif_fatal_tx_err(vif); 778 return -EINVAL; 779 } 780 781 skb_shinfo(skb)->gso_size = gso->u.gso.size; 782 /* gso_segs will be calculated later */ 783 784 return 0; 785} 786 787static int checksum_setup(struct xenvif_queue *queue, struct sk_buff *skb) 788{ 789 bool recalculate_partial_csum = false; 790 791 /* A GSO SKB must be CHECKSUM_PARTIAL. However some buggy 792 * peers can fail to set NETRXF_csum_blank when sending a GSO 793 * frame. In this case force the SKB to CHECKSUM_PARTIAL and 794 * recalculate the partial checksum. 795 */ 796 if (skb->ip_summed != CHECKSUM_PARTIAL && skb_is_gso(skb)) { 797 queue->stats.rx_gso_checksum_fixup++; 798 skb->ip_summed = CHECKSUM_PARTIAL; 799 recalculate_partial_csum = true; 800 } 801 802 /* A non-CHECKSUM_PARTIAL SKB does not require setup. */ 803 if (skb->ip_summed != CHECKSUM_PARTIAL) 804 return 0; 805 806 return skb_checksum_setup(skb, recalculate_partial_csum); 807} 808 809static bool tx_credit_exceeded(struct xenvif_queue *queue, unsigned size) 810{ 811 u64 now = get_jiffies_64(); 812 u64 next_credit = queue->credit_window_start + 813 msecs_to_jiffies(queue->credit_usec / 1000); 814 815 /* Timer could already be pending in rare cases. */ 816 if (timer_pending(&queue->credit_timeout)) { 817 queue->rate_limited = true; 818 return true; 819 } 820 821 /* Passed the point where we can replenish credit? */ 822 if (time_after_eq64(now, next_credit)) { 823 queue->credit_window_start = now; 824 tx_add_credit(queue); 825 } 826 827 /* Still too big to send right now? Set a callback. */ 828 if (size > queue->remaining_credit) { 829 mod_timer(&queue->credit_timeout, 830 next_credit); 831 queue->credit_window_start = next_credit; 832 queue->rate_limited = true; 833 834 return true; 835 } 836 837 return false; 838} 839 840/* No locking is required in xenvif_mcast_add/del() as they are 841 * only ever invoked from NAPI poll. An RCU list is used because 842 * xenvif_mcast_match() is called asynchronously, during start_xmit. 843 */ 844 845static int xenvif_mcast_add(struct xenvif *vif, const u8 *addr) 846{ 847 struct xenvif_mcast_addr *mcast; 848 849 if (vif->fe_mcast_count == XEN_NETBK_MCAST_MAX) { 850 if (net_ratelimit()) 851 netdev_err(vif->dev, 852 "Too many multicast addresses\n"); 853 return -ENOSPC; 854 } 855 856 mcast = kzalloc(sizeof(*mcast), GFP_ATOMIC); 857 if (!mcast) 858 return -ENOMEM; 859 860 ether_addr_copy(mcast->addr, addr); 861 list_add_tail_rcu(&mcast->entry, &vif->fe_mcast_addr); 862 vif->fe_mcast_count++; 863 864 return 0; 865} 866 867static void xenvif_mcast_del(struct xenvif *vif, const u8 *addr) 868{ 869 struct xenvif_mcast_addr *mcast; 870 871 list_for_each_entry_rcu(mcast, &vif->fe_mcast_addr, entry) { 872 if (ether_addr_equal(addr, mcast->addr)) { 873 --vif->fe_mcast_count; 874 list_del_rcu(&mcast->entry); 875 kfree_rcu(mcast, rcu); 876 break; 877 } 878 } 879} 880 881bool xenvif_mcast_match(struct xenvif *vif, const u8 *addr) 882{ 883 struct xenvif_mcast_addr *mcast; 884 885 rcu_read_lock(); 886 list_for_each_entry_rcu(mcast, &vif->fe_mcast_addr, entry) { 887 if (ether_addr_equal(addr, mcast->addr)) { 888 rcu_read_unlock(); 889 return true; 890 } 891 } 892 rcu_read_unlock(); 893 894 return false; 895} 896 897void xenvif_mcast_addr_list_free(struct xenvif *vif) 898{ 899 /* No need for locking or RCU here. NAPI poll and TX queue 900 * are stopped. 901 */ 902 while (!list_empty(&vif->fe_mcast_addr)) { 903 struct xenvif_mcast_addr *mcast; 904 905 mcast = list_first_entry(&vif->fe_mcast_addr, 906 struct xenvif_mcast_addr, 907 entry); 908 --vif->fe_mcast_count; 909 list_del(&mcast->entry); 910 kfree(mcast); 911 } 912} 913 914static void xenvif_tx_build_gops(struct xenvif_queue *queue, 915 int budget, 916 unsigned *copy_ops, 917 unsigned *map_ops) 918{ 919 struct sk_buff *skb, *nskb; 920 int ret; 921 unsigned int frag_overflow; 922 923 while (skb_queue_len(&queue->tx_queue) < budget) { 924 struct xen_netif_tx_request txreq; 925 struct xen_netif_tx_request txfrags[XEN_NETBK_LEGACY_SLOTS_MAX]; 926 struct xen_netif_extra_info extras[XEN_NETIF_EXTRA_TYPE_MAX-1]; 927 unsigned int extra_count; 928 u16 pending_idx; 929 RING_IDX idx; 930 int work_to_do; 931 unsigned int data_len; 932 pending_ring_idx_t index; 933 934 if (queue->tx.sring->req_prod - queue->tx.req_cons > 935 XEN_NETIF_TX_RING_SIZE) { 936 netdev_err(queue->vif->dev, 937 "Impossible number of requests. " 938 "req_prod %d, req_cons %d, size %ld\n", 939 queue->tx.sring->req_prod, queue->tx.req_cons, 940 XEN_NETIF_TX_RING_SIZE); 941 xenvif_fatal_tx_err(queue->vif); 942 break; 943 } 944 945 work_to_do = RING_HAS_UNCONSUMED_REQUESTS(&queue->tx); 946 if (!work_to_do) 947 break; 948 949 idx = queue->tx.req_cons; 950 rmb(); /* Ensure that we see the request before we copy it. */ 951 RING_COPY_REQUEST(&queue->tx, idx, &txreq); 952 953 /* Credit-based scheduling. */ 954 if (txreq.size > queue->remaining_credit && 955 tx_credit_exceeded(queue, txreq.size)) 956 break; 957 958 queue->remaining_credit -= txreq.size; 959 960 work_to_do--; 961 queue->tx.req_cons = ++idx; 962 963 memset(extras, 0, sizeof(extras)); 964 extra_count = 0; 965 if (txreq.flags & XEN_NETTXF_extra_info) { 966 work_to_do = xenvif_get_extras(queue, extras, 967 &extra_count, 968 work_to_do); 969 idx = queue->tx.req_cons; 970 if (unlikely(work_to_do < 0)) 971 break; 972 } 973 974 if (extras[XEN_NETIF_EXTRA_TYPE_MCAST_ADD - 1].type) { 975 struct xen_netif_extra_info *extra; 976 977 extra = &extras[XEN_NETIF_EXTRA_TYPE_MCAST_ADD - 1]; 978 ret = xenvif_mcast_add(queue->vif, extra->u.mcast.addr); 979 980 make_tx_response(queue, &txreq, extra_count, 981 (ret == 0) ? 982 XEN_NETIF_RSP_OKAY : 983 XEN_NETIF_RSP_ERROR); 984 continue; 985 } 986 987 if (extras[XEN_NETIF_EXTRA_TYPE_MCAST_DEL - 1].type) { 988 struct xen_netif_extra_info *extra; 989 990 extra = &extras[XEN_NETIF_EXTRA_TYPE_MCAST_DEL - 1]; 991 xenvif_mcast_del(queue->vif, extra->u.mcast.addr); 992 993 make_tx_response(queue, &txreq, extra_count, 994 XEN_NETIF_RSP_OKAY); 995 continue; 996 } 997 998 data_len = (txreq.size > XEN_NETBACK_TX_COPY_LEN) ? 999 XEN_NETBACK_TX_COPY_LEN : txreq.size; 1000 1001 ret = xenvif_count_requests(queue, &txreq, extra_count, 1002 txfrags, work_to_do); 1003 1004 if (unlikely(ret < 0)) 1005 break; 1006 1007 idx += ret; 1008 1009 if (unlikely(txreq.size < ETH_HLEN)) { 1010 netdev_dbg(queue->vif->dev, 1011 "Bad packet size: %d\n", txreq.size); 1012 xenvif_tx_err(queue, &txreq, extra_count, idx); 1013 break; 1014 } 1015 1016 /* No crossing a page as the payload mustn't fragment. */ 1017 if (unlikely((txreq.offset + txreq.size) > XEN_PAGE_SIZE)) { 1018 netdev_err(queue->vif->dev, "Cross page boundary, txreq.offset: %u, size: %u\n", 1019 txreq.offset, txreq.size); 1020 xenvif_fatal_tx_err(queue->vif); 1021 break; 1022 } 1023 1024 index = pending_index(queue->pending_cons); 1025 pending_idx = queue->pending_ring[index]; 1026 1027 if (ret >= XEN_NETBK_LEGACY_SLOTS_MAX - 1 && data_len < txreq.size) 1028 data_len = txreq.size; 1029 1030 skb = xenvif_alloc_skb(data_len); 1031 if (unlikely(skb == NULL)) { 1032 netdev_dbg(queue->vif->dev, 1033 "Can't allocate a skb in start_xmit.\n"); 1034 xenvif_tx_err(queue, &txreq, extra_count, idx); 1035 break; 1036 } 1037 1038 skb_shinfo(skb)->nr_frags = ret; 1039 /* At this point shinfo->nr_frags is in fact the number of 1040 * slots, which can be as large as XEN_NETBK_LEGACY_SLOTS_MAX. 1041 */ 1042 frag_overflow = 0; 1043 nskb = NULL; 1044 if (skb_shinfo(skb)->nr_frags > MAX_SKB_FRAGS) { 1045 frag_overflow = skb_shinfo(skb)->nr_frags - MAX_SKB_FRAGS; 1046 BUG_ON(frag_overflow > MAX_SKB_FRAGS); 1047 skb_shinfo(skb)->nr_frags = MAX_SKB_FRAGS; 1048 nskb = xenvif_alloc_skb(0); 1049 if (unlikely(nskb == NULL)) { 1050 skb_shinfo(skb)->nr_frags = 0; 1051 kfree_skb(skb); 1052 xenvif_tx_err(queue, &txreq, extra_count, idx); 1053 if (net_ratelimit()) 1054 netdev_err(queue->vif->dev, 1055 "Can't allocate the frag_list skb.\n"); 1056 break; 1057 } 1058 } 1059 1060 if (extras[XEN_NETIF_EXTRA_TYPE_GSO - 1].type) { 1061 struct xen_netif_extra_info *gso; 1062 gso = &extras[XEN_NETIF_EXTRA_TYPE_GSO - 1]; 1063 1064 if (xenvif_set_skb_gso(queue->vif, skb, gso)) { 1065 /* Failure in xenvif_set_skb_gso is fatal. */ 1066 skb_shinfo(skb)->nr_frags = 0; 1067 kfree_skb(skb); 1068 kfree_skb(nskb); 1069 break; 1070 } 1071 } 1072 1073 if (extras[XEN_NETIF_EXTRA_TYPE_HASH - 1].type) { 1074 struct xen_netif_extra_info *extra; 1075 enum pkt_hash_types type = PKT_HASH_TYPE_NONE; 1076 1077 extra = &extras[XEN_NETIF_EXTRA_TYPE_HASH - 1]; 1078 1079 switch (extra->u.hash.type) { 1080 case _XEN_NETIF_CTRL_HASH_TYPE_IPV4: 1081 case _XEN_NETIF_CTRL_HASH_TYPE_IPV6: 1082 type = PKT_HASH_TYPE_L3; 1083 break; 1084 1085 case _XEN_NETIF_CTRL_HASH_TYPE_IPV4_TCP: 1086 case _XEN_NETIF_CTRL_HASH_TYPE_IPV6_TCP: 1087 type = PKT_HASH_TYPE_L4; 1088 break; 1089 1090 default: 1091 break; 1092 } 1093 1094 if (type != PKT_HASH_TYPE_NONE) 1095 skb_set_hash(skb, 1096 *(u32 *)extra->u.hash.value, 1097 type); 1098 } 1099 1100 xenvif_get_requests(queue, skb, &txreq, txfrags, copy_ops, 1101 map_ops, frag_overflow, nskb, extra_count, 1102 data_len); 1103 1104 __skb_queue_tail(&queue->tx_queue, skb); 1105 1106 queue->tx.req_cons = idx; 1107 1108 if ((*map_ops >= ARRAY_SIZE(queue->tx_map_ops)) || 1109 (*copy_ops >= ARRAY_SIZE(queue->tx_copy_ops))) 1110 break; 1111 } 1112 1113 return; 1114} 1115 1116/* Consolidate skb with a frag_list into a brand new one with local pages on 1117 * frags. Returns 0 or -ENOMEM if can't allocate new pages. 1118 */ 1119static int xenvif_handle_frag_list(struct xenvif_queue *queue, struct sk_buff *skb) 1120{ 1121 unsigned int offset = skb_headlen(skb); 1122 skb_frag_t frags[MAX_SKB_FRAGS]; 1123 int i, f; 1124 struct ubuf_info *uarg; 1125 struct sk_buff *nskb = skb_shinfo(skb)->frag_list; 1126 1127 queue->stats.tx_zerocopy_sent += 2; 1128 queue->stats.tx_frag_overflow++; 1129 1130 xenvif_fill_frags(queue, nskb); 1131 /* Subtract frags size, we will correct it later */ 1132 skb->truesize -= skb->data_len; 1133 skb->len += nskb->len; 1134 skb->data_len += nskb->len; 1135 1136 /* create a brand new frags array and coalesce there */ 1137 for (i = 0; offset < skb->len; i++) { 1138 struct page *page; 1139 unsigned int len; 1140 1141 BUG_ON(i >= MAX_SKB_FRAGS); 1142 page = alloc_page(GFP_ATOMIC); 1143 if (!page) { 1144 int j; 1145 skb->truesize += skb->data_len; 1146 for (j = 0; j < i; j++) 1147 put_page(skb_frag_page(&frags[j])); 1148 return -ENOMEM; 1149 } 1150 1151 if (offset + PAGE_SIZE < skb->len) 1152 len = PAGE_SIZE; 1153 else 1154 len = skb->len - offset; 1155 if (skb_copy_bits(skb, offset, page_address(page), len)) 1156 BUG(); 1157 1158 offset += len; 1159 __skb_frag_set_page(&frags[i], page); 1160 skb_frag_off_set(&frags[i], 0); 1161 skb_frag_size_set(&frags[i], len); 1162 } 1163 1164 /* Release all the original (foreign) frags. */ 1165 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) 1166 skb_frag_unref(skb, f); 1167 uarg = skb_shinfo(skb)->destructor_arg; 1168 /* increase inflight counter to offset decrement in callback */ 1169 atomic_inc(&queue->inflight_packets); 1170 uarg->callback(uarg, true); 1171 skb_shinfo(skb)->destructor_arg = NULL; 1172 1173 /* Fill the skb with the new (local) frags. */ 1174 memcpy(skb_shinfo(skb)->frags, frags, i * sizeof(skb_frag_t)); 1175 skb_shinfo(skb)->nr_frags = i; 1176 skb->truesize += i * PAGE_SIZE; 1177 1178 return 0; 1179} 1180 1181static int xenvif_tx_submit(struct xenvif_queue *queue) 1182{ 1183 struct gnttab_map_grant_ref *gop_map = queue->tx_map_ops; 1184 struct gnttab_copy *gop_copy = queue->tx_copy_ops; 1185 struct sk_buff *skb; 1186 int work_done = 0; 1187 1188 while ((skb = __skb_dequeue(&queue->tx_queue)) != NULL) { 1189 struct xen_netif_tx_request *txp; 1190 u16 pending_idx; 1191 1192 pending_idx = copy_pending_idx(skb, 0); 1193 txp = &queue->pending_tx_info[pending_idx].req; 1194 1195 /* Check the remap error code. */ 1196 if (unlikely(xenvif_tx_check_gop(queue, skb, &gop_map, &gop_copy))) { 1197 /* If there was an error, xenvif_tx_check_gop is 1198 * expected to release all the frags which were mapped, 1199 * so kfree_skb shouldn't do it again 1200 */ 1201 skb_shinfo(skb)->nr_frags = 0; 1202 if (skb_has_frag_list(skb)) { 1203 struct sk_buff *nskb = 1204 skb_shinfo(skb)->frag_list; 1205 skb_shinfo(nskb)->nr_frags = 0; 1206 } 1207 kfree_skb(skb); 1208 continue; 1209 } 1210 1211 if (txp->flags & XEN_NETTXF_csum_blank) 1212 skb->ip_summed = CHECKSUM_PARTIAL; 1213 else if (txp->flags & XEN_NETTXF_data_validated) 1214 skb->ip_summed = CHECKSUM_UNNECESSARY; 1215 1216 xenvif_fill_frags(queue, skb); 1217 1218 if (unlikely(skb_has_frag_list(skb))) { 1219 struct sk_buff *nskb = skb_shinfo(skb)->frag_list; 1220 xenvif_skb_zerocopy_prepare(queue, nskb); 1221 if (xenvif_handle_frag_list(queue, skb)) { 1222 if (net_ratelimit()) 1223 netdev_err(queue->vif->dev, 1224 "Not enough memory to consolidate frag_list!\n"); 1225 xenvif_skb_zerocopy_prepare(queue, skb); 1226 kfree_skb(skb); 1227 continue; 1228 } 1229 /* Copied all the bits from the frag list -- free it. */ 1230 skb_frag_list_init(skb); 1231 kfree_skb(nskb); 1232 } 1233 1234 skb->dev = queue->vif->dev; 1235 skb->protocol = eth_type_trans(skb, skb->dev); 1236 skb_reset_network_header(skb); 1237 1238 if (checksum_setup(queue, skb)) { 1239 netdev_dbg(queue->vif->dev, 1240 "Can't setup checksum in net_tx_action\n"); 1241 /* We have to set this flag to trigger the callback */ 1242 if (skb_shinfo(skb)->destructor_arg) 1243 xenvif_skb_zerocopy_prepare(queue, skb); 1244 kfree_skb(skb); 1245 continue; 1246 } 1247 1248 skb_probe_transport_header(skb); 1249 1250 /* If the packet is GSO then we will have just set up the 1251 * transport header offset in checksum_setup so it's now 1252 * straightforward to calculate gso_segs. 1253 */ 1254 if (skb_is_gso(skb)) { 1255 int mss, hdrlen; 1256 1257 /* GSO implies having the L4 header. */ 1258 WARN_ON_ONCE(!skb_transport_header_was_set(skb)); 1259 if (unlikely(!skb_transport_header_was_set(skb))) { 1260 kfree_skb(skb); 1261 continue; 1262 } 1263 1264 mss = skb_shinfo(skb)->gso_size; 1265 hdrlen = skb_transport_header(skb) - 1266 skb_mac_header(skb) + 1267 tcp_hdrlen(skb); 1268 1269 skb_shinfo(skb)->gso_segs = 1270 DIV_ROUND_UP(skb->len - hdrlen, mss); 1271 } 1272 1273 queue->stats.rx_bytes += skb->len; 1274 queue->stats.rx_packets++; 1275 1276 work_done++; 1277 1278 /* Set this flag right before netif_receive_skb, otherwise 1279 * someone might think this packet already left netback, and 1280 * do a skb_copy_ubufs while we are still in control of the 1281 * skb. E.g. the __pskb_pull_tail earlier can do such thing. 1282 */ 1283 if (skb_shinfo(skb)->destructor_arg) { 1284 xenvif_skb_zerocopy_prepare(queue, skb); 1285 queue->stats.tx_zerocopy_sent++; 1286 } 1287 1288 netif_receive_skb(skb); 1289 } 1290 1291 return work_done; 1292} 1293 1294void xenvif_zerocopy_callback(struct ubuf_info *ubuf, bool zerocopy_success) 1295{ 1296 unsigned long flags; 1297 pending_ring_idx_t index; 1298 struct xenvif_queue *queue = ubuf_to_queue(ubuf); 1299 1300 /* This is the only place where we grab this lock, to protect callbacks 1301 * from each other. 1302 */ 1303 spin_lock_irqsave(&queue->callback_lock, flags); 1304 do { 1305 u16 pending_idx = ubuf->desc; 1306 ubuf = (struct ubuf_info *) ubuf->ctx; 1307 BUG_ON(queue->dealloc_prod - queue->dealloc_cons >= 1308 MAX_PENDING_REQS); 1309 index = pending_index(queue->dealloc_prod); 1310 queue->dealloc_ring[index] = pending_idx; 1311 /* Sync with xenvif_tx_dealloc_action: 1312 * insert idx then incr producer. 1313 */ 1314 smp_wmb(); 1315 queue->dealloc_prod++; 1316 } while (ubuf); 1317 spin_unlock_irqrestore(&queue->callback_lock, flags); 1318 1319 if (likely(zerocopy_success)) 1320 queue->stats.tx_zerocopy_success++; 1321 else 1322 queue->stats.tx_zerocopy_fail++; 1323 xenvif_skb_zerocopy_complete(queue); 1324} 1325 1326static inline void xenvif_tx_dealloc_action(struct xenvif_queue *queue) 1327{ 1328 struct gnttab_unmap_grant_ref *gop; 1329 pending_ring_idx_t dc, dp; 1330 u16 pending_idx, pending_idx_release[MAX_PENDING_REQS]; 1331 unsigned int i = 0; 1332 1333 dc = queue->dealloc_cons; 1334 gop = queue->tx_unmap_ops; 1335 1336 /* Free up any grants we have finished using */ 1337 do { 1338 dp = queue->dealloc_prod; 1339 1340 /* Ensure we see all indices enqueued by all 1341 * xenvif_zerocopy_callback(). 1342 */ 1343 smp_rmb(); 1344 1345 while (dc != dp) { 1346 BUG_ON(gop - queue->tx_unmap_ops >= MAX_PENDING_REQS); 1347 pending_idx = 1348 queue->dealloc_ring[pending_index(dc++)]; 1349 1350 pending_idx_release[gop - queue->tx_unmap_ops] = 1351 pending_idx; 1352 queue->pages_to_unmap[gop - queue->tx_unmap_ops] = 1353 queue->mmap_pages[pending_idx]; 1354 gnttab_set_unmap_op(gop, 1355 idx_to_kaddr(queue, pending_idx), 1356 GNTMAP_host_map, 1357 queue->grant_tx_handle[pending_idx]); 1358 xenvif_grant_handle_reset(queue, pending_idx); 1359 ++gop; 1360 } 1361 1362 } while (dp != queue->dealloc_prod); 1363 1364 queue->dealloc_cons = dc; 1365 1366 if (gop - queue->tx_unmap_ops > 0) { 1367 int ret; 1368 ret = gnttab_unmap_refs(queue->tx_unmap_ops, 1369 NULL, 1370 queue->pages_to_unmap, 1371 gop - queue->tx_unmap_ops); 1372 if (ret) { 1373 netdev_err(queue->vif->dev, "Unmap fail: nr_ops %tu ret %d\n", 1374 gop - queue->tx_unmap_ops, ret); 1375 for (i = 0; i < gop - queue->tx_unmap_ops; ++i) { 1376 if (gop[i].status != GNTST_okay) 1377 netdev_err(queue->vif->dev, 1378 " host_addr: 0x%llx handle: 0x%x status: %d\n", 1379 gop[i].host_addr, 1380 gop[i].handle, 1381 gop[i].status); 1382 } 1383 BUG(); 1384 } 1385 } 1386 1387 for (i = 0; i < gop - queue->tx_unmap_ops; ++i) 1388 xenvif_idx_release(queue, pending_idx_release[i], 1389 XEN_NETIF_RSP_OKAY); 1390} 1391 1392 1393/* Called after netfront has transmitted */ 1394int xenvif_tx_action(struct xenvif_queue *queue, int budget) 1395{ 1396 unsigned nr_mops = 0, nr_cops = 0; 1397 int work_done, ret; 1398 1399 if (unlikely(!tx_work_todo(queue))) 1400 return 0; 1401 1402 xenvif_tx_build_gops(queue, budget, &nr_cops, &nr_mops); 1403 1404 if (nr_cops == 0) 1405 return 0; 1406 1407 gnttab_batch_copy(queue->tx_copy_ops, nr_cops); 1408 if (nr_mops != 0) { 1409 ret = gnttab_map_refs(queue->tx_map_ops, 1410 NULL, 1411 queue->pages_to_map, 1412 nr_mops); 1413 if (ret) { 1414 unsigned int i; 1415 1416 netdev_err(queue->vif->dev, "Map fail: nr %u ret %d\n", 1417 nr_mops, ret); 1418 for (i = 0; i < nr_mops; ++i) 1419 WARN_ON_ONCE(queue->tx_map_ops[i].status == 1420 GNTST_okay); 1421 } 1422 } 1423 1424 work_done = xenvif_tx_submit(queue); 1425 1426 return work_done; 1427} 1428 1429static void _make_tx_response(struct xenvif_queue *queue, 1430 const struct xen_netif_tx_request *txp, 1431 unsigned int extra_count, 1432 s8 status) 1433{ 1434 RING_IDX i = queue->tx.rsp_prod_pvt; 1435 struct xen_netif_tx_response *resp; 1436 1437 resp = RING_GET_RESPONSE(&queue->tx, i); 1438 resp->id = txp->id; 1439 resp->status = status; 1440 1441 while (extra_count-- != 0) 1442 RING_GET_RESPONSE(&queue->tx, ++i)->status = XEN_NETIF_RSP_NULL; 1443 1444 queue->tx.rsp_prod_pvt = ++i; 1445} 1446 1447static void push_tx_responses(struct xenvif_queue *queue) 1448{ 1449 int notify; 1450 1451 RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&queue->tx, notify); 1452 if (notify) 1453 notify_remote_via_irq(queue->tx_irq); 1454} 1455 1456static void xenvif_idx_release(struct xenvif_queue *queue, u16 pending_idx, 1457 s8 status) 1458{ 1459 struct pending_tx_info *pending_tx_info; 1460 pending_ring_idx_t index; 1461 unsigned long flags; 1462 1463 pending_tx_info = &queue->pending_tx_info[pending_idx]; 1464 1465 spin_lock_irqsave(&queue->response_lock, flags); 1466 1467 _make_tx_response(queue, &pending_tx_info->req, 1468 pending_tx_info->extra_count, status); 1469 1470 /* Release the pending index before pusing the Tx response so 1471 * its available before a new Tx request is pushed by the 1472 * frontend. 1473 */ 1474 index = pending_index(queue->pending_prod++); 1475 queue->pending_ring[index] = pending_idx; 1476 1477 push_tx_responses(queue); 1478 1479 spin_unlock_irqrestore(&queue->response_lock, flags); 1480} 1481 1482static void make_tx_response(struct xenvif_queue *queue, 1483 const struct xen_netif_tx_request *txp, 1484 unsigned int extra_count, 1485 s8 status) 1486{ 1487 unsigned long flags; 1488 1489 spin_lock_irqsave(&queue->response_lock, flags); 1490 1491 _make_tx_response(queue, txp, extra_count, status); 1492 push_tx_responses(queue); 1493 1494 spin_unlock_irqrestore(&queue->response_lock, flags); 1495} 1496 1497static void xenvif_idx_unmap(struct xenvif_queue *queue, u16 pending_idx) 1498{ 1499 int ret; 1500 struct gnttab_unmap_grant_ref tx_unmap_op; 1501 1502 gnttab_set_unmap_op(&tx_unmap_op, 1503 idx_to_kaddr(queue, pending_idx), 1504 GNTMAP_host_map, 1505 queue->grant_tx_handle[pending_idx]); 1506 xenvif_grant_handle_reset(queue, pending_idx); 1507 1508 ret = gnttab_unmap_refs(&tx_unmap_op, NULL, 1509 &queue->mmap_pages[pending_idx], 1); 1510 if (ret) { 1511 netdev_err(queue->vif->dev, 1512 "Unmap fail: ret: %d pending_idx: %d host_addr: %llx handle: 0x%x status: %d\n", 1513 ret, 1514 pending_idx, 1515 tx_unmap_op.host_addr, 1516 tx_unmap_op.handle, 1517 tx_unmap_op.status); 1518 BUG(); 1519 } 1520} 1521 1522static inline int tx_work_todo(struct xenvif_queue *queue) 1523{ 1524 if (likely(RING_HAS_UNCONSUMED_REQUESTS(&queue->tx))) 1525 return 1; 1526 1527 return 0; 1528} 1529 1530static inline bool tx_dealloc_work_todo(struct xenvif_queue *queue) 1531{ 1532 return queue->dealloc_cons != queue->dealloc_prod; 1533} 1534 1535void xenvif_unmap_frontend_data_rings(struct xenvif_queue *queue) 1536{ 1537 if (queue->tx.sring) 1538 xenbus_unmap_ring_vfree(xenvif_to_xenbus_device(queue->vif), 1539 queue->tx.sring); 1540 if (queue->rx.sring) 1541 xenbus_unmap_ring_vfree(xenvif_to_xenbus_device(queue->vif), 1542 queue->rx.sring); 1543} 1544 1545int xenvif_map_frontend_data_rings(struct xenvif_queue *queue, 1546 grant_ref_t tx_ring_ref, 1547 grant_ref_t rx_ring_ref) 1548{ 1549 void *addr; 1550 struct xen_netif_tx_sring *txs; 1551 struct xen_netif_rx_sring *rxs; 1552 RING_IDX rsp_prod, req_prod; 1553 int err = -ENOMEM; 1554 1555 err = xenbus_map_ring_valloc(xenvif_to_xenbus_device(queue->vif), 1556 &tx_ring_ref, 1, &addr); 1557 if (err) 1558 goto err; 1559 1560 txs = (struct xen_netif_tx_sring *)addr; 1561 rsp_prod = READ_ONCE(txs->rsp_prod); 1562 req_prod = READ_ONCE(txs->req_prod); 1563 1564 BACK_RING_ATTACH(&queue->tx, txs, rsp_prod, XEN_PAGE_SIZE); 1565 1566 err = -EIO; 1567 if (req_prod - rsp_prod > RING_SIZE(&queue->tx)) 1568 goto err; 1569 1570 err = xenbus_map_ring_valloc(xenvif_to_xenbus_device(queue->vif), 1571 &rx_ring_ref, 1, &addr); 1572 if (err) 1573 goto err; 1574 1575 rxs = (struct xen_netif_rx_sring *)addr; 1576 rsp_prod = READ_ONCE(rxs->rsp_prod); 1577 req_prod = READ_ONCE(rxs->req_prod); 1578 1579 BACK_RING_ATTACH(&queue->rx, rxs, rsp_prod, XEN_PAGE_SIZE); 1580 1581 err = -EIO; 1582 if (req_prod - rsp_prod > RING_SIZE(&queue->rx)) 1583 goto err; 1584 1585 return 0; 1586 1587err: 1588 xenvif_unmap_frontend_data_rings(queue); 1589 return err; 1590} 1591 1592static bool xenvif_dealloc_kthread_should_stop(struct xenvif_queue *queue) 1593{ 1594 /* Dealloc thread must remain running until all inflight 1595 * packets complete. 1596 */ 1597 return kthread_should_stop() && 1598 !atomic_read(&queue->inflight_packets); 1599} 1600 1601int xenvif_dealloc_kthread(void *data) 1602{ 1603 struct xenvif_queue *queue = data; 1604 1605 for (;;) { 1606 wait_event_interruptible(queue->dealloc_wq, 1607 tx_dealloc_work_todo(queue) || 1608 xenvif_dealloc_kthread_should_stop(queue)); 1609 if (xenvif_dealloc_kthread_should_stop(queue)) 1610 break; 1611 1612 xenvif_tx_dealloc_action(queue); 1613 cond_resched(); 1614 } 1615 1616 /* Unmap anything remaining*/ 1617 if (tx_dealloc_work_todo(queue)) 1618 xenvif_tx_dealloc_action(queue); 1619 1620 return 0; 1621} 1622 1623static void make_ctrl_response(struct xenvif *vif, 1624 const struct xen_netif_ctrl_request *req, 1625 u32 status, u32 data) 1626{ 1627 RING_IDX idx = vif->ctrl.rsp_prod_pvt; 1628 struct xen_netif_ctrl_response rsp = { 1629 .id = req->id, 1630 .type = req->type, 1631 .status = status, 1632 .data = data, 1633 }; 1634 1635 *RING_GET_RESPONSE(&vif->ctrl, idx) = rsp; 1636 vif->ctrl.rsp_prod_pvt = ++idx; 1637} 1638 1639static void push_ctrl_response(struct xenvif *vif) 1640{ 1641 int notify; 1642 1643 RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&vif->ctrl, notify); 1644 if (notify) 1645 notify_remote_via_irq(vif->ctrl_irq); 1646} 1647 1648static void process_ctrl_request(struct xenvif *vif, 1649 const struct xen_netif_ctrl_request *req) 1650{ 1651 u32 status = XEN_NETIF_CTRL_STATUS_NOT_SUPPORTED; 1652 u32 data = 0; 1653 1654 switch (req->type) { 1655 case XEN_NETIF_CTRL_TYPE_SET_HASH_ALGORITHM: 1656 status = xenvif_set_hash_alg(vif, req->data[0]); 1657 break; 1658 1659 case XEN_NETIF_CTRL_TYPE_GET_HASH_FLAGS: 1660 status = xenvif_get_hash_flags(vif, &data); 1661 break; 1662 1663 case XEN_NETIF_CTRL_TYPE_SET_HASH_FLAGS: 1664 status = xenvif_set_hash_flags(vif, req->data[0]); 1665 break; 1666 1667 case XEN_NETIF_CTRL_TYPE_SET_HASH_KEY: 1668 status = xenvif_set_hash_key(vif, req->data[0], 1669 req->data[1]); 1670 break; 1671 1672 case XEN_NETIF_CTRL_TYPE_GET_HASH_MAPPING_SIZE: 1673 status = XEN_NETIF_CTRL_STATUS_SUCCESS; 1674 data = XEN_NETBK_MAX_HASH_MAPPING_SIZE; 1675 break; 1676 1677 case XEN_NETIF_CTRL_TYPE_SET_HASH_MAPPING_SIZE: 1678 status = xenvif_set_hash_mapping_size(vif, 1679 req->data[0]); 1680 break; 1681 1682 case XEN_NETIF_CTRL_TYPE_SET_HASH_MAPPING: 1683 status = xenvif_set_hash_mapping(vif, req->data[0], 1684 req->data[1], 1685 req->data[2]); 1686 break; 1687 1688 default: 1689 break; 1690 } 1691 1692 make_ctrl_response(vif, req, status, data); 1693 push_ctrl_response(vif); 1694} 1695 1696static void xenvif_ctrl_action(struct xenvif *vif) 1697{ 1698 for (;;) { 1699 RING_IDX req_prod, req_cons; 1700 1701 req_prod = vif->ctrl.sring->req_prod; 1702 req_cons = vif->ctrl.req_cons; 1703 1704 /* Make sure we can see requests before we process them. */ 1705 rmb(); 1706 1707 if (req_cons == req_prod) 1708 break; 1709 1710 while (req_cons != req_prod) { 1711 struct xen_netif_ctrl_request req; 1712 1713 RING_COPY_REQUEST(&vif->ctrl, req_cons, &req); 1714 req_cons++; 1715 1716 process_ctrl_request(vif, &req); 1717 } 1718 1719 vif->ctrl.req_cons = req_cons; 1720 vif->ctrl.sring->req_event = req_cons + 1; 1721 } 1722} 1723 1724static bool xenvif_ctrl_work_todo(struct xenvif *vif) 1725{ 1726 if (likely(RING_HAS_UNCONSUMED_REQUESTS(&vif->ctrl))) 1727 return true; 1728 1729 return false; 1730} 1731 1732irqreturn_t xenvif_ctrl_irq_fn(int irq, void *data) 1733{ 1734 struct xenvif *vif = data; 1735 unsigned int eoi_flag = XEN_EOI_FLAG_SPURIOUS; 1736 1737 while (xenvif_ctrl_work_todo(vif)) { 1738 xenvif_ctrl_action(vif); 1739 eoi_flag = 0; 1740 } 1741 1742 xen_irq_lateeoi(irq, eoi_flag); 1743 1744 return IRQ_HANDLED; 1745} 1746 1747static int __init netback_init(void) 1748{ 1749 int rc = 0; 1750 1751 if (!xen_domain()) 1752 return -ENODEV; 1753 1754 /* Allow as many queues as there are CPUs but max. 8 if user has not 1755 * specified a value. 1756 */ 1757 if (xenvif_max_queues == 0) 1758 xenvif_max_queues = min_t(unsigned int, MAX_QUEUES_DEFAULT, 1759 num_online_cpus()); 1760 1761 if (fatal_skb_slots < XEN_NETBK_LEGACY_SLOTS_MAX) { 1762 pr_info("fatal_skb_slots too small (%d), bump it to XEN_NETBK_LEGACY_SLOTS_MAX (%d)\n", 1763 fatal_skb_slots, XEN_NETBK_LEGACY_SLOTS_MAX); 1764 fatal_skb_slots = XEN_NETBK_LEGACY_SLOTS_MAX; 1765 } 1766 1767 rc = xenvif_xenbus_init(); 1768 if (rc) 1769 goto failed_init; 1770 1771#ifdef CONFIG_DEBUG_FS 1772 xen_netback_dbg_root = debugfs_create_dir("xen-netback", NULL); 1773#endif /* CONFIG_DEBUG_FS */ 1774 1775 return 0; 1776 1777failed_init: 1778 return rc; 1779} 1780 1781module_init(netback_init); 1782 1783static void __exit netback_fini(void) 1784{ 1785#ifdef CONFIG_DEBUG_FS 1786 debugfs_remove_recursive(xen_netback_dbg_root); 1787#endif /* CONFIG_DEBUG_FS */ 1788 xenvif_xenbus_fini(); 1789} 1790module_exit(netback_fini); 1791 1792MODULE_LICENSE("Dual BSD/GPL"); 1793MODULE_ALIAS("xen-backend:vif"); 1794