1// SPDX-License-Identifier: GPL-2.0-only 2/* Copyright (c) 2017 Facebook 3 */ 4#include <linux/bpf.h> 5#include <linux/slab.h> 6#include <linux/vmalloc.h> 7#include <linux/etherdevice.h> 8#include <linux/filter.h> 9#include <linux/sched/signal.h> 10#include <net/bpf_sk_storage.h> 11#include <net/sock.h> 12#include <net/tcp.h> 13#include <linux/error-injection.h> 14#include <linux/smp.h> 15 16#define CREATE_TRACE_POINTS 17#include <trace/events/bpf_test_run.h> 18 19static int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat, 20 u32 *retval, u32 *time, bool xdp) 21{ 22 struct bpf_prog_array_item item = {.prog = prog}; 23 struct bpf_run_ctx *old_ctx; 24 struct bpf_cg_run_ctx run_ctx; 25 enum bpf_cgroup_storage_type stype; 26 u64 time_start, time_spent = 0; 27 int ret = 0; 28 u32 i; 29 30 for_each_cgroup_storage_type(stype) { 31 item.cgroup_storage[stype] = bpf_cgroup_storage_alloc(prog, stype); 32 if (IS_ERR(item.cgroup_storage[stype])) { 33 item.cgroup_storage[stype] = NULL; 34 for_each_cgroup_storage_type(stype) 35 bpf_cgroup_storage_free(item.cgroup_storage[stype]); 36 return -ENOMEM; 37 } 38 } 39 40 if (!repeat) 41 repeat = 1; 42 43 rcu_read_lock(); 44 migrate_disable(); 45 time_start = ktime_get_ns(); 46 old_ctx = bpf_set_run_ctx(&run_ctx.run_ctx); 47 for (i = 0; i < repeat; i++) { 48 run_ctx.prog_item = &item; 49 50 if (xdp) 51 *retval = bpf_prog_run_xdp(prog, ctx); 52 else 53 *retval = BPF_PROG_RUN(prog, ctx); 54 55 if (signal_pending(current)) { 56 ret = -EINTR; 57 break; 58 } 59 60 if (need_resched()) { 61 time_spent += ktime_get_ns() - time_start; 62 migrate_enable(); 63 rcu_read_unlock(); 64 65 cond_resched(); 66 67 rcu_read_lock(); 68 migrate_disable(); 69 time_start = ktime_get_ns(); 70 } 71 } 72 bpf_reset_run_ctx(old_ctx); 73 time_spent += ktime_get_ns() - time_start; 74 migrate_enable(); 75 rcu_read_unlock(); 76 77 do_div(time_spent, repeat); 78 *time = time_spent > U32_MAX ? U32_MAX : (u32)time_spent; 79 80 for_each_cgroup_storage_type(stype) 81 bpf_cgroup_storage_free(item.cgroup_storage[stype]); 82 83 return ret; 84} 85 86static int bpf_test_finish(const union bpf_attr *kattr, 87 union bpf_attr __user *uattr, const void *data, 88 u32 size, u32 retval, u32 duration) 89{ 90 void __user *data_out = u64_to_user_ptr(kattr->test.data_out); 91 int err = -EFAULT; 92 u32 copy_size = size; 93 94 /* Clamp copy if the user has provided a size hint, but copy the full 95 * buffer if not to retain old behaviour. 96 */ 97 if (kattr->test.data_size_out && 98 copy_size > kattr->test.data_size_out) { 99 copy_size = kattr->test.data_size_out; 100 err = -ENOSPC; 101 } 102 103 if (data_out && copy_to_user(data_out, data, copy_size)) 104 goto out; 105 if (copy_to_user(&uattr->test.data_size_out, &size, sizeof(size))) 106 goto out; 107 if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval))) 108 goto out; 109 if (copy_to_user(&uattr->test.duration, &duration, sizeof(duration))) 110 goto out; 111 if (err != -ENOSPC) 112 err = 0; 113out: 114 trace_bpf_test_finish(&err); 115 return err; 116} 117 118/* Integer types of various sizes and pointer combinations cover variety of 119 * architecture dependent calling conventions. 7+ can be supported in the 120 * future. 121 */ 122__diag_push(); 123__diag_ignore(GCC, 8, "-Wmissing-prototypes", 124 "Global functions as their definitions will be in vmlinux BTF"); 125int noinline bpf_fentry_test1(int a) 126{ 127 return a + 1; 128} 129 130int noinline bpf_fentry_test2(int a, u64 b) 131{ 132 return a + b; 133} 134 135int noinline bpf_fentry_test3(char a, int b, u64 c) 136{ 137 return a + b + c; 138} 139 140int noinline bpf_fentry_test4(void *a, char b, int c, u64 d) 141{ 142 return (long)a + b + c + d; 143} 144 145int noinline bpf_fentry_test5(u64 a, void *b, short c, int d, u64 e) 146{ 147 return a + (long)b + c + d + e; 148} 149 150int noinline bpf_fentry_test6(u64 a, void *b, short c, int d, void *e, u64 f) 151{ 152 return a + (long)b + c + d + (long)e + f; 153} 154 155struct bpf_fentry_test_t { 156 struct bpf_fentry_test_t *a; 157}; 158 159int noinline bpf_fentry_test7(struct bpf_fentry_test_t *arg) 160{ 161 return (long)arg; 162} 163 164int noinline bpf_fentry_test8(struct bpf_fentry_test_t *arg) 165{ 166 return (long)arg->a; 167} 168 169int noinline bpf_modify_return_test(int a, int *b) 170{ 171 *b += 1; 172 return a + *b; 173} 174__diag_pop(); 175 176ALLOW_ERROR_INJECTION(bpf_modify_return_test, ERRNO); 177 178static void *bpf_test_init(const union bpf_attr *kattr, u32 size, 179 u32 headroom, u32 tailroom) 180{ 181 void __user *data_in = u64_to_user_ptr(kattr->test.data_in); 182 u32 user_size = kattr->test.data_size_in; 183 void *data; 184 185 if (size < ETH_HLEN || size > PAGE_SIZE - headroom - tailroom) 186 return ERR_PTR(-EINVAL); 187 188 if (user_size > size) 189 return ERR_PTR(-EMSGSIZE); 190 191 size = SKB_DATA_ALIGN(size); 192 data = kzalloc(size + headroom + tailroom, GFP_USER); 193 if (!data) 194 return ERR_PTR(-ENOMEM); 195 196 if (copy_from_user(data + headroom, data_in, user_size)) { 197 kfree(data); 198 return ERR_PTR(-EFAULT); 199 } 200 201 return data; 202} 203 204int bpf_prog_test_run_tracing(struct bpf_prog *prog, 205 const union bpf_attr *kattr, 206 union bpf_attr __user *uattr) 207{ 208 struct bpf_fentry_test_t arg = {}; 209 u16 side_effect = 0, ret = 0; 210 int b = 2, err = -EFAULT; 211 u32 retval = 0; 212 213 if (kattr->test.flags || kattr->test.cpu) 214 return -EINVAL; 215 216 switch (prog->expected_attach_type) { 217 case BPF_TRACE_FENTRY: 218 case BPF_TRACE_FEXIT: 219 if (bpf_fentry_test1(1) != 2 || 220 bpf_fentry_test2(2, 3) != 5 || 221 bpf_fentry_test3(4, 5, 6) != 15 || 222 bpf_fentry_test4((void *)7, 8, 9, 10) != 34 || 223 bpf_fentry_test5(11, (void *)12, 13, 14, 15) != 65 || 224 bpf_fentry_test6(16, (void *)17, 18, 19, (void *)20, 21) != 111 || 225 bpf_fentry_test7((struct bpf_fentry_test_t *)0) != 0 || 226 bpf_fentry_test8(&arg) != 0) 227 goto out; 228 break; 229 case BPF_MODIFY_RETURN: 230 ret = bpf_modify_return_test(1, &b); 231 if (b != 2) 232 side_effect = 1; 233 break; 234 default: 235 goto out; 236 } 237 238 retval = ((u32)side_effect << 16) | ret; 239 if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval))) 240 goto out; 241 242 err = 0; 243out: 244 trace_bpf_test_finish(&err); 245 return err; 246} 247 248struct bpf_raw_tp_test_run_info { 249 struct bpf_prog *prog; 250 void *ctx; 251 u32 retval; 252}; 253 254static void 255__bpf_prog_test_run_raw_tp(void *data) 256{ 257 struct bpf_raw_tp_test_run_info *info = data; 258 259 rcu_read_lock(); 260 info->retval = BPF_PROG_RUN(info->prog, info->ctx); 261 rcu_read_unlock(); 262} 263 264int bpf_prog_test_run_raw_tp(struct bpf_prog *prog, 265 const union bpf_attr *kattr, 266 union bpf_attr __user *uattr) 267{ 268 void __user *ctx_in = u64_to_user_ptr(kattr->test.ctx_in); 269 __u32 ctx_size_in = kattr->test.ctx_size_in; 270 struct bpf_raw_tp_test_run_info info; 271 int cpu = kattr->test.cpu, err = 0; 272 int current_cpu; 273 274 /* doesn't support data_in/out, ctx_out, duration, or repeat */ 275 if (kattr->test.data_in || kattr->test.data_out || 276 kattr->test.ctx_out || kattr->test.duration || 277 kattr->test.repeat) 278 return -EINVAL; 279 280 if (ctx_size_in < prog->aux->max_ctx_offset || 281 ctx_size_in > MAX_BPF_FUNC_ARGS * sizeof(u64)) 282 return -EINVAL; 283 284 if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 && cpu != 0) 285 return -EINVAL; 286 287 if (ctx_size_in) { 288 info.ctx = kzalloc(ctx_size_in, GFP_USER); 289 if (!info.ctx) 290 return -ENOMEM; 291 if (copy_from_user(info.ctx, ctx_in, ctx_size_in)) { 292 err = -EFAULT; 293 goto out; 294 } 295 } else { 296 info.ctx = NULL; 297 } 298 299 info.prog = prog; 300 301 current_cpu = get_cpu(); 302 if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 || 303 cpu == current_cpu) { 304 __bpf_prog_test_run_raw_tp(&info); 305 } else if (cpu >= nr_cpu_ids || !cpu_online(cpu)) { 306 /* smp_call_function_single() also checks cpu_online() 307 * after csd_lock(). However, since cpu is from user 308 * space, let's do an extra quick check to filter out 309 * invalid value before smp_call_function_single(). 310 */ 311 err = -ENXIO; 312 } else { 313 err = smp_call_function_single(cpu, __bpf_prog_test_run_raw_tp, 314 &info, 1); 315 } 316 put_cpu(); 317 318 if (!err && 319 copy_to_user(&uattr->test.retval, &info.retval, sizeof(u32))) 320 err = -EFAULT; 321 322out: 323 kfree(info.ctx); 324 return err; 325} 326 327static void *bpf_ctx_init(const union bpf_attr *kattr, u32 max_size) 328{ 329 void __user *data_in = u64_to_user_ptr(kattr->test.ctx_in); 330 void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out); 331 u32 size = kattr->test.ctx_size_in; 332 void *data; 333 int err; 334 335 if (!data_in && !data_out) 336 return NULL; 337 338 data = kzalloc(max_size, GFP_USER); 339 if (!data) 340 return ERR_PTR(-ENOMEM); 341 342 if (data_in) { 343 err = bpf_check_uarg_tail_zero(data_in, max_size, size); 344 if (err) { 345 kfree(data); 346 return ERR_PTR(err); 347 } 348 349 size = min_t(u32, max_size, size); 350 if (copy_from_user(data, data_in, size)) { 351 kfree(data); 352 return ERR_PTR(-EFAULT); 353 } 354 } 355 return data; 356} 357 358static int bpf_ctx_finish(const union bpf_attr *kattr, 359 union bpf_attr __user *uattr, const void *data, 360 u32 size) 361{ 362 void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out); 363 int err = -EFAULT; 364 u32 copy_size = size; 365 366 if (!data || !data_out) 367 return 0; 368 369 if (copy_size > kattr->test.ctx_size_out) { 370 copy_size = kattr->test.ctx_size_out; 371 err = -ENOSPC; 372 } 373 374 if (copy_to_user(data_out, data, copy_size)) 375 goto out; 376 if (copy_to_user(&uattr->test.ctx_size_out, &size, sizeof(size))) 377 goto out; 378 if (err != -ENOSPC) 379 err = 0; 380out: 381 return err; 382} 383 384/** 385 * range_is_zero - test whether buffer is initialized 386 * @buf: buffer to check 387 * @from: check from this position 388 * @to: check up until (excluding) this position 389 * 390 * This function returns true if the there is a non-zero byte 391 * in the buf in the range [from,to). 392 */ 393static inline bool range_is_zero(void *buf, size_t from, size_t to) 394{ 395 return !memchr_inv((u8 *)buf + from, 0, to - from); 396} 397 398static int convert___skb_to_skb(struct sk_buff *skb, struct __sk_buff *__skb) 399{ 400 struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb; 401 402 if (!__skb) 403 return 0; 404 405 /* make sure the fields we don't use are zeroed */ 406 if (!range_is_zero(__skb, 0, offsetof(struct __sk_buff, mark))) 407 return -EINVAL; 408 409 /* mark is allowed */ 410 411 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, mark), 412 offsetof(struct __sk_buff, priority))) 413 return -EINVAL; 414 415 /* priority is allowed */ 416 417 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, priority), 418 offsetof(struct __sk_buff, ifindex))) 419 return -EINVAL; 420 421 /* ifindex is allowed */ 422 423 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, ifindex), 424 offsetof(struct __sk_buff, cb))) 425 return -EINVAL; 426 427 /* cb is allowed */ 428 429 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, cb), 430 offsetof(struct __sk_buff, tstamp))) 431 return -EINVAL; 432 433 /* tstamp is allowed */ 434 /* wire_len is allowed */ 435 /* gso_segs is allowed */ 436 437 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, gso_segs), 438 offsetof(struct __sk_buff, gso_size))) 439 return -EINVAL; 440 441 /* gso_size is allowed */ 442 443 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, gso_size), 444 sizeof(struct __sk_buff))) 445 return -EINVAL; 446 447 skb->mark = __skb->mark; 448 skb->priority = __skb->priority; 449 skb->tstamp = __skb->tstamp; 450 memcpy(&cb->data, __skb->cb, QDISC_CB_PRIV_LEN); 451 452 if (__skb->wire_len == 0) { 453 cb->pkt_len = skb->len; 454 } else { 455 if (__skb->wire_len < skb->len || 456 __skb->wire_len > GSO_MAX_SIZE) 457 return -EINVAL; 458 cb->pkt_len = __skb->wire_len; 459 } 460 461 if (__skb->gso_segs > GSO_MAX_SEGS) 462 return -EINVAL; 463 skb_shinfo(skb)->gso_segs = __skb->gso_segs; 464 skb_shinfo(skb)->gso_size = __skb->gso_size; 465 466 return 0; 467} 468 469static void convert_skb_to___skb(struct sk_buff *skb, struct __sk_buff *__skb) 470{ 471 struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb; 472 473 if (!__skb) 474 return; 475 476 __skb->mark = skb->mark; 477 __skb->priority = skb->priority; 478 __skb->ifindex = skb->dev->ifindex; 479 __skb->tstamp = skb->tstamp; 480 memcpy(__skb->cb, &cb->data, QDISC_CB_PRIV_LEN); 481 __skb->wire_len = cb->pkt_len; 482 __skb->gso_segs = skb_shinfo(skb)->gso_segs; 483} 484 485static struct proto bpf_dummy_proto = { 486 .name = "bpf_dummy", 487 .owner = THIS_MODULE, 488 .obj_size = sizeof(struct sock), 489}; 490 491int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr, 492 union bpf_attr __user *uattr) 493{ 494 bool is_l2 = false, is_direct_pkt_access = false; 495 struct net *net = current->nsproxy->net_ns; 496 struct net_device *dev = net->loopback_dev; 497 u32 size = kattr->test.data_size_in; 498 u32 repeat = kattr->test.repeat; 499 struct __sk_buff *ctx = NULL; 500 u32 retval, duration; 501 int hh_len = ETH_HLEN; 502 struct sk_buff *skb; 503 struct sock *sk; 504 void *data; 505 int ret; 506 507 if (kattr->test.flags || kattr->test.cpu) 508 return -EINVAL; 509 510 data = bpf_test_init(kattr, size, NET_SKB_PAD + NET_IP_ALIGN, 511 SKB_DATA_ALIGN(sizeof(struct skb_shared_info))); 512 if (IS_ERR(data)) 513 return PTR_ERR(data); 514 515 ctx = bpf_ctx_init(kattr, sizeof(struct __sk_buff)); 516 if (IS_ERR(ctx)) { 517 kfree(data); 518 return PTR_ERR(ctx); 519 } 520 521 switch (prog->type) { 522 case BPF_PROG_TYPE_SCHED_CLS: 523 case BPF_PROG_TYPE_SCHED_ACT: 524 is_l2 = true; 525 fallthrough; 526 case BPF_PROG_TYPE_LWT_IN: 527 case BPF_PROG_TYPE_LWT_OUT: 528 case BPF_PROG_TYPE_LWT_XMIT: 529 is_direct_pkt_access = true; 530 break; 531 default: 532 break; 533 } 534 535 sk = sk_alloc(net, AF_UNSPEC, GFP_USER, &bpf_dummy_proto, 1); 536 if (!sk) { 537 kfree(data); 538 kfree(ctx); 539 return -ENOMEM; 540 } 541 sock_init_data(NULL, sk); 542 543 skb = build_skb(data, 0); 544 if (!skb) { 545 kfree(data); 546 kfree(ctx); 547 sk_free(sk); 548 return -ENOMEM; 549 } 550 skb->sk = sk; 551 552 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN); 553 __skb_put(skb, size); 554 if (ctx && ctx->ifindex > 1) { 555 dev = dev_get_by_index(net, ctx->ifindex); 556 if (!dev) { 557 ret = -ENODEV; 558 goto out; 559 } 560 } 561 skb->protocol = eth_type_trans(skb, dev); 562 skb_reset_network_header(skb); 563 564 switch (skb->protocol) { 565 case htons(ETH_P_IP): 566 sk->sk_family = AF_INET; 567 if (sizeof(struct iphdr) <= skb_headlen(skb)) { 568 sk->sk_rcv_saddr = ip_hdr(skb)->saddr; 569 sk->sk_daddr = ip_hdr(skb)->daddr; 570 } 571 break; 572#if IS_ENABLED(CONFIG_IPV6) 573 case htons(ETH_P_IPV6): 574 sk->sk_family = AF_INET6; 575 if (sizeof(struct ipv6hdr) <= skb_headlen(skb)) { 576 sk->sk_v6_rcv_saddr = ipv6_hdr(skb)->saddr; 577 sk->sk_v6_daddr = ipv6_hdr(skb)->daddr; 578 } 579 break; 580#endif 581 default: 582 break; 583 } 584 585 if (is_l2) 586 __skb_push(skb, hh_len); 587 if (is_direct_pkt_access) 588 bpf_compute_data_pointers(skb); 589 ret = convert___skb_to_skb(skb, ctx); 590 if (ret) 591 goto out; 592 ret = bpf_test_run(prog, skb, repeat, &retval, &duration, false); 593 if (ret) 594 goto out; 595 if (!is_l2) { 596 if (skb_headroom(skb) < hh_len) { 597 int nhead = HH_DATA_ALIGN(hh_len - skb_headroom(skb)); 598 599 if (pskb_expand_head(skb, nhead, 0, GFP_USER)) { 600 ret = -ENOMEM; 601 goto out; 602 } 603 } 604 memset(__skb_push(skb, hh_len), 0, hh_len); 605 } 606 convert_skb_to___skb(skb, ctx); 607 608 size = skb->len; 609 /* bpf program can never convert linear skb to non-linear */ 610 if (WARN_ON_ONCE(skb_is_nonlinear(skb))) 611 size = skb_headlen(skb); 612 ret = bpf_test_finish(kattr, uattr, skb->data, size, retval, duration); 613 if (!ret) 614 ret = bpf_ctx_finish(kattr, uattr, ctx, 615 sizeof(struct __sk_buff)); 616out: 617 if (dev && dev != net->loopback_dev) 618 dev_put(dev); 619 kfree_skb(skb); 620 sk_free(sk); 621 kfree(ctx); 622 return ret; 623} 624 625int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr, 626 union bpf_attr __user *uattr) 627{ 628 u32 tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 629 u32 headroom = XDP_PACKET_HEADROOM; 630 u32 size = kattr->test.data_size_in; 631 u32 repeat = kattr->test.repeat; 632 struct netdev_rx_queue *rxqueue; 633 struct xdp_buff xdp = {}; 634 u32 retval, duration; 635 u32 max_data_sz; 636 void *data; 637 int ret; 638 639 if (prog->expected_attach_type == BPF_XDP_DEVMAP || 640 prog->expected_attach_type == BPF_XDP_CPUMAP) 641 return -EINVAL; 642 if (kattr->test.ctx_in || kattr->test.ctx_out) 643 return -EINVAL; 644 645 /* XDP have extra tailroom as (most) drivers use full page */ 646 max_data_sz = 4096 - headroom - tailroom; 647 648 data = bpf_test_init(kattr, max_data_sz, headroom, tailroom); 649 if (IS_ERR(data)) 650 return PTR_ERR(data); 651 652 xdp.data_hard_start = data; 653 xdp.data = data + headroom; 654 xdp.data_meta = xdp.data; 655 xdp.data_end = xdp.data + size; 656 xdp.frame_sz = headroom + max_data_sz + tailroom; 657 658 rxqueue = __netif_get_rx_queue(current->nsproxy->net_ns->loopback_dev, 0); 659 xdp.rxq = &rxqueue->xdp_rxq; 660 bpf_prog_change_xdp(NULL, prog); 661 ret = bpf_test_run(prog, &xdp, repeat, &retval, &duration, true); 662 if (ret) 663 goto out; 664 if (xdp.data != data + headroom || xdp.data_end != xdp.data + size) 665 size = xdp.data_end - xdp.data; 666 ret = bpf_test_finish(kattr, uattr, xdp.data, size, retval, duration); 667out: 668 bpf_prog_change_xdp(prog, NULL); 669 kfree(data); 670 return ret; 671} 672 673static int verify_user_bpf_flow_keys(struct bpf_flow_keys *ctx) 674{ 675 /* make sure the fields we don't use are zeroed */ 676 if (!range_is_zero(ctx, 0, offsetof(struct bpf_flow_keys, flags))) 677 return -EINVAL; 678 679 /* flags is allowed */ 680 681 if (!range_is_zero(ctx, offsetofend(struct bpf_flow_keys, flags), 682 sizeof(struct bpf_flow_keys))) 683 return -EINVAL; 684 685 return 0; 686} 687 688int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog, 689 const union bpf_attr *kattr, 690 union bpf_attr __user *uattr) 691{ 692 u32 size = kattr->test.data_size_in; 693 struct bpf_flow_dissector ctx = {}; 694 u32 repeat = kattr->test.repeat; 695 struct bpf_flow_keys *user_ctx; 696 struct bpf_flow_keys flow_keys; 697 u64 time_start, time_spent = 0; 698 const struct ethhdr *eth; 699 unsigned int flags = 0; 700 u32 retval, duration; 701 void *data; 702 int ret; 703 u32 i; 704 705 if (prog->type != BPF_PROG_TYPE_FLOW_DISSECTOR) 706 return -EINVAL; 707 708 if (kattr->test.flags || kattr->test.cpu) 709 return -EINVAL; 710 711 if (size < ETH_HLEN) 712 return -EINVAL; 713 714 data = bpf_test_init(kattr, size, 0, 0); 715 if (IS_ERR(data)) 716 return PTR_ERR(data); 717 718 eth = (struct ethhdr *)data; 719 720 if (!repeat) 721 repeat = 1; 722 723 user_ctx = bpf_ctx_init(kattr, sizeof(struct bpf_flow_keys)); 724 if (IS_ERR(user_ctx)) { 725 kfree(data); 726 return PTR_ERR(user_ctx); 727 } 728 if (user_ctx) { 729 ret = verify_user_bpf_flow_keys(user_ctx); 730 if (ret) 731 goto out; 732 flags = user_ctx->flags; 733 } 734 735 ctx.flow_keys = &flow_keys; 736 ctx.data = data; 737 ctx.data_end = (__u8 *)data + size; 738 739 rcu_read_lock(); 740 preempt_disable(); 741 time_start = ktime_get_ns(); 742 for (i = 0; i < repeat; i++) { 743 retval = bpf_flow_dissect(prog, &ctx, eth->h_proto, ETH_HLEN, 744 size, flags); 745 746 if (signal_pending(current)) { 747 preempt_enable(); 748 rcu_read_unlock(); 749 750 ret = -EINTR; 751 goto out; 752 } 753 754 if (need_resched()) { 755 time_spent += ktime_get_ns() - time_start; 756 preempt_enable(); 757 rcu_read_unlock(); 758 759 cond_resched(); 760 761 rcu_read_lock(); 762 preempt_disable(); 763 time_start = ktime_get_ns(); 764 } 765 } 766 time_spent += ktime_get_ns() - time_start; 767 preempt_enable(); 768 rcu_read_unlock(); 769 770 do_div(time_spent, repeat); 771 duration = time_spent > U32_MAX ? U32_MAX : (u32)time_spent; 772 773 ret = bpf_test_finish(kattr, uattr, &flow_keys, sizeof(flow_keys), 774 retval, duration); 775 if (!ret) 776 ret = bpf_ctx_finish(kattr, uattr, user_ctx, 777 sizeof(struct bpf_flow_keys)); 778 779out: 780 kfree(user_ctx); 781 kfree(data); 782 return ret; 783} 784