1// SPDX-License-Identifier: GPL-2.0-only 2/* Copyright (c) 2017 Facebook 3 */ 4#include <linux/bpf.h> 5#include <linux/btf.h> 6#include <linux/btf_ids.h> 7#include <linux/slab.h> 8#include <linux/init.h> 9#include <linux/vmalloc.h> 10#include <linux/etherdevice.h> 11#include <linux/filter.h> 12#include <linux/rcupdate_trace.h> 13#include <linux/sched/signal.h> 14#include <net/bpf_sk_storage.h> 15#include <net/sock.h> 16#include <net/tcp.h> 17#include <net/net_namespace.h> 18#include <net/page_pool/helpers.h> 19#include <linux/error-injection.h> 20#include <linux/smp.h> 21#include <linux/sock_diag.h> 22#include <linux/netfilter.h> 23#include <net/netdev_rx_queue.h> 24#include <net/xdp.h> 25#include <net/netfilter/nf_bpf_link.h> 26 27#define CREATE_TRACE_POINTS 28#include <trace/events/bpf_test_run.h> 29 30struct bpf_test_timer { 31 enum { NO_PREEMPT, NO_MIGRATE } mode; 32 u32 i; 33 u64 time_start, time_spent; 34}; 35 36static void bpf_test_timer_enter(struct bpf_test_timer *t) 37 __acquires(rcu) 38{ 39 rcu_read_lock(); 40 if (t->mode == NO_PREEMPT) 41 preempt_disable(); 42 else 43 migrate_disable(); 44 45 t->time_start = ktime_get_ns(); 46} 47 48static void bpf_test_timer_leave(struct bpf_test_timer *t) 49 __releases(rcu) 50{ 51 t->time_start = 0; 52 53 if (t->mode == NO_PREEMPT) 54 preempt_enable(); 55 else 56 migrate_enable(); 57 rcu_read_unlock(); 58} 59 60static bool bpf_test_timer_continue(struct bpf_test_timer *t, int iterations, 61 u32 repeat, int *err, u32 *duration) 62 __must_hold(rcu) 63{ 64 t->i += iterations; 65 if (t->i >= repeat) { 66 /* We're done. */ 67 t->time_spent += ktime_get_ns() - t->time_start; 68 do_div(t->time_spent, t->i); 69 *duration = t->time_spent > U32_MAX ? U32_MAX : (u32)t->time_spent; 70 *err = 0; 71 goto reset; 72 } 73 74 if (signal_pending(current)) { 75 /* During iteration: we've been cancelled, abort. */ 76 *err = -EINTR; 77 goto reset; 78 } 79 80 if (need_resched()) { 81 /* During iteration: we need to reschedule between runs. */ 82 t->time_spent += ktime_get_ns() - t->time_start; 83 bpf_test_timer_leave(t); 84 cond_resched(); 85 bpf_test_timer_enter(t); 86 } 87 88 /* Do another round. */ 89 return true; 90 91reset: 92 t->i = 0; 93 return false; 94} 95 96/* We put this struct at the head of each page with a context and frame 97 * initialised when the page is allocated, so we don't have to do this on each 98 * repetition of the test run. 99 */ 100struct xdp_page_head { 101 struct xdp_buff orig_ctx; 102 struct xdp_buff ctx; 103 union { 104 /* ::data_hard_start starts here */ 105 DECLARE_FLEX_ARRAY(struct xdp_frame, frame); 106 DECLARE_FLEX_ARRAY(u8, data); 107 }; 108}; 109 110struct xdp_test_data { 111 struct xdp_buff *orig_ctx; 112 struct xdp_rxq_info rxq; 113 struct net_device *dev; 114 struct page_pool *pp; 115 struct xdp_frame **frames; 116 struct sk_buff **skbs; 117 struct xdp_mem_info mem; 118 u32 batch_size; 119 u32 frame_cnt; 120}; 121 122/* tools/testing/selftests/bpf/prog_tests/xdp_do_redirect.c:%MAX_PKT_SIZE 123 * must be updated accordingly this gets changed, otherwise BPF selftests 124 * will fail. 125 */ 126#define TEST_XDP_FRAME_SIZE (PAGE_SIZE - sizeof(struct xdp_page_head)) 127#define TEST_XDP_MAX_BATCH 256 128 129static void xdp_test_run_init_page(struct page *page, void *arg) 130{ 131 struct xdp_page_head *head = phys_to_virt(page_to_phys(page)); 132 struct xdp_buff *new_ctx, *orig_ctx; 133 u32 headroom = XDP_PACKET_HEADROOM; 134 struct xdp_test_data *xdp = arg; 135 size_t frm_len, meta_len; 136 struct xdp_frame *frm; 137 void *data; 138 139 orig_ctx = xdp->orig_ctx; 140 frm_len = orig_ctx->data_end - orig_ctx->data_meta; 141 meta_len = orig_ctx->data - orig_ctx->data_meta; 142 headroom -= meta_len; 143 144 new_ctx = &head->ctx; 145 frm = head->frame; 146 data = head->data; 147 memcpy(data + headroom, orig_ctx->data_meta, frm_len); 148 149 xdp_init_buff(new_ctx, TEST_XDP_FRAME_SIZE, &xdp->rxq); 150 xdp_prepare_buff(new_ctx, data, headroom, frm_len, true); 151 new_ctx->data = new_ctx->data_meta + meta_len; 152 153 xdp_update_frame_from_buff(new_ctx, frm); 154 frm->mem = new_ctx->rxq->mem; 155 156 memcpy(&head->orig_ctx, new_ctx, sizeof(head->orig_ctx)); 157} 158 159static int xdp_test_run_setup(struct xdp_test_data *xdp, struct xdp_buff *orig_ctx) 160{ 161 struct page_pool *pp; 162 int err = -ENOMEM; 163 struct page_pool_params pp_params = { 164 .order = 0, 165 .flags = 0, 166 .pool_size = xdp->batch_size, 167 .nid = NUMA_NO_NODE, 168 .init_callback = xdp_test_run_init_page, 169 .init_arg = xdp, 170 }; 171 172 xdp->frames = kvmalloc_array(xdp->batch_size, sizeof(void *), GFP_KERNEL); 173 if (!xdp->frames) 174 return -ENOMEM; 175 176 xdp->skbs = kvmalloc_array(xdp->batch_size, sizeof(void *), GFP_KERNEL); 177 if (!xdp->skbs) 178 goto err_skbs; 179 180 pp = page_pool_create(&pp_params); 181 if (IS_ERR(pp)) { 182 err = PTR_ERR(pp); 183 goto err_pp; 184 } 185 186 /* will copy 'mem.id' into pp->xdp_mem_id */ 187 err = xdp_reg_mem_model(&xdp->mem, MEM_TYPE_PAGE_POOL, pp); 188 if (err) 189 goto err_mmodel; 190 191 xdp->pp = pp; 192 193 /* We create a 'fake' RXQ referencing the original dev, but with an 194 * xdp_mem_info pointing to our page_pool 195 */ 196 xdp_rxq_info_reg(&xdp->rxq, orig_ctx->rxq->dev, 0, 0); 197 xdp->rxq.mem.type = MEM_TYPE_PAGE_POOL; 198 xdp->rxq.mem.id = pp->xdp_mem_id; 199 xdp->dev = orig_ctx->rxq->dev; 200 xdp->orig_ctx = orig_ctx; 201 202 return 0; 203 204err_mmodel: 205 page_pool_destroy(pp); 206err_pp: 207 kvfree(xdp->skbs); 208err_skbs: 209 kvfree(xdp->frames); 210 return err; 211} 212 213static void xdp_test_run_teardown(struct xdp_test_data *xdp) 214{ 215 xdp_unreg_mem_model(&xdp->mem); 216 page_pool_destroy(xdp->pp); 217 kfree(xdp->frames); 218 kfree(xdp->skbs); 219} 220 221static bool frame_was_changed(const struct xdp_page_head *head) 222{ 223 /* xdp_scrub_frame() zeroes the data pointer, flags is the last field, 224 * i.e. has the highest chances to be overwritten. If those two are 225 * untouched, it's most likely safe to skip the context reset. 226 */ 227 return head->frame->data != head->orig_ctx.data || 228 head->frame->flags != head->orig_ctx.flags; 229} 230 231static bool ctx_was_changed(struct xdp_page_head *head) 232{ 233 return head->orig_ctx.data != head->ctx.data || 234 head->orig_ctx.data_meta != head->ctx.data_meta || 235 head->orig_ctx.data_end != head->ctx.data_end; 236} 237 238static void reset_ctx(struct xdp_page_head *head) 239{ 240 if (likely(!frame_was_changed(head) && !ctx_was_changed(head))) 241 return; 242 243 head->ctx.data = head->orig_ctx.data; 244 head->ctx.data_meta = head->orig_ctx.data_meta; 245 head->ctx.data_end = head->orig_ctx.data_end; 246 xdp_update_frame_from_buff(&head->ctx, head->frame); 247} 248 249static int xdp_recv_frames(struct xdp_frame **frames, int nframes, 250 struct sk_buff **skbs, 251 struct net_device *dev) 252{ 253 gfp_t gfp = __GFP_ZERO | GFP_ATOMIC; 254 int i, n; 255 LIST_HEAD(list); 256 257 n = kmem_cache_alloc_bulk(skbuff_cache, gfp, nframes, (void **)skbs); 258 if (unlikely(n == 0)) { 259 for (i = 0; i < nframes; i++) 260 xdp_return_frame(frames[i]); 261 return -ENOMEM; 262 } 263 264 for (i = 0; i < nframes; i++) { 265 struct xdp_frame *xdpf = frames[i]; 266 struct sk_buff *skb = skbs[i]; 267 268 skb = __xdp_build_skb_from_frame(xdpf, skb, dev); 269 if (!skb) { 270 xdp_return_frame(xdpf); 271 continue; 272 } 273 274 list_add_tail(&skb->list, &list); 275 } 276 netif_receive_skb_list(&list); 277 278 return 0; 279} 280 281static int xdp_test_run_batch(struct xdp_test_data *xdp, struct bpf_prog *prog, 282 u32 repeat) 283{ 284 struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info); 285 int err = 0, act, ret, i, nframes = 0, batch_sz; 286 struct xdp_frame **frames = xdp->frames; 287 struct xdp_page_head *head; 288 struct xdp_frame *frm; 289 bool redirect = false; 290 struct xdp_buff *ctx; 291 struct page *page; 292 293 batch_sz = min_t(u32, repeat, xdp->batch_size); 294 295 local_bh_disable(); 296 xdp_set_return_frame_no_direct(); 297 298 for (i = 0; i < batch_sz; i++) { 299 page = page_pool_dev_alloc_pages(xdp->pp); 300 if (!page) { 301 err = -ENOMEM; 302 goto out; 303 } 304 305 head = phys_to_virt(page_to_phys(page)); 306 reset_ctx(head); 307 ctx = &head->ctx; 308 frm = head->frame; 309 xdp->frame_cnt++; 310 311 act = bpf_prog_run_xdp(prog, ctx); 312 313 /* if program changed pkt bounds we need to update the xdp_frame */ 314 if (unlikely(ctx_was_changed(head))) { 315 ret = xdp_update_frame_from_buff(ctx, frm); 316 if (ret) { 317 xdp_return_buff(ctx); 318 continue; 319 } 320 } 321 322 switch (act) { 323 case XDP_TX: 324 /* we can't do a real XDP_TX since we're not in the 325 * driver, so turn it into a REDIRECT back to the same 326 * index 327 */ 328 ri->tgt_index = xdp->dev->ifindex; 329 ri->map_id = INT_MAX; 330 ri->map_type = BPF_MAP_TYPE_UNSPEC; 331 fallthrough; 332 case XDP_REDIRECT: 333 redirect = true; 334 ret = xdp_do_redirect_frame(xdp->dev, ctx, frm, prog); 335 if (ret) 336 xdp_return_buff(ctx); 337 break; 338 case XDP_PASS: 339 frames[nframes++] = frm; 340 break; 341 default: 342 bpf_warn_invalid_xdp_action(NULL, prog, act); 343 fallthrough; 344 case XDP_DROP: 345 xdp_return_buff(ctx); 346 break; 347 } 348 } 349 350out: 351 if (redirect) 352 xdp_do_flush(); 353 if (nframes) { 354 ret = xdp_recv_frames(frames, nframes, xdp->skbs, xdp->dev); 355 if (ret) 356 err = ret; 357 } 358 359 xdp_clear_return_frame_no_direct(); 360 local_bh_enable(); 361 return err; 362} 363 364static int bpf_test_run_xdp_live(struct bpf_prog *prog, struct xdp_buff *ctx, 365 u32 repeat, u32 batch_size, u32 *time) 366 367{ 368 struct xdp_test_data xdp = { .batch_size = batch_size }; 369 struct bpf_test_timer t = { .mode = NO_MIGRATE }; 370 int ret; 371 372 if (!repeat) 373 repeat = 1; 374 375 ret = xdp_test_run_setup(&xdp, ctx); 376 if (ret) 377 return ret; 378 379 bpf_test_timer_enter(&t); 380 do { 381 xdp.frame_cnt = 0; 382 ret = xdp_test_run_batch(&xdp, prog, repeat - t.i); 383 if (unlikely(ret < 0)) 384 break; 385 } while (bpf_test_timer_continue(&t, xdp.frame_cnt, repeat, &ret, time)); 386 bpf_test_timer_leave(&t); 387 388 xdp_test_run_teardown(&xdp); 389 return ret; 390} 391 392static int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat, 393 u32 *retval, u32 *time, bool xdp) 394{ 395 struct bpf_prog_array_item item = {.prog = prog}; 396 struct bpf_run_ctx *old_ctx; 397 struct bpf_cg_run_ctx run_ctx; 398 struct bpf_test_timer t = { NO_MIGRATE }; 399 enum bpf_cgroup_storage_type stype; 400 int ret; 401 402 for_each_cgroup_storage_type(stype) { 403 item.cgroup_storage[stype] = bpf_cgroup_storage_alloc(prog, stype); 404 if (IS_ERR(item.cgroup_storage[stype])) { 405 item.cgroup_storage[stype] = NULL; 406 for_each_cgroup_storage_type(stype) 407 bpf_cgroup_storage_free(item.cgroup_storage[stype]); 408 return -ENOMEM; 409 } 410 } 411 412 if (!repeat) 413 repeat = 1; 414 415 bpf_test_timer_enter(&t); 416 old_ctx = bpf_set_run_ctx(&run_ctx.run_ctx); 417 do { 418 run_ctx.prog_item = &item; 419 local_bh_disable(); 420 if (xdp) 421 *retval = bpf_prog_run_xdp(prog, ctx); 422 else 423 *retval = bpf_prog_run(prog, ctx); 424 local_bh_enable(); 425 } while (bpf_test_timer_continue(&t, 1, repeat, &ret, time)); 426 bpf_reset_run_ctx(old_ctx); 427 bpf_test_timer_leave(&t); 428 429 for_each_cgroup_storage_type(stype) 430 bpf_cgroup_storage_free(item.cgroup_storage[stype]); 431 432 return ret; 433} 434 435static int bpf_test_finish(const union bpf_attr *kattr, 436 union bpf_attr __user *uattr, const void *data, 437 struct skb_shared_info *sinfo, u32 size, 438 u32 retval, u32 duration) 439{ 440 void __user *data_out = u64_to_user_ptr(kattr->test.data_out); 441 int err = -EFAULT; 442 u32 copy_size = size; 443 444 /* Clamp copy if the user has provided a size hint, but copy the full 445 * buffer if not to retain old behaviour. 446 */ 447 if (kattr->test.data_size_out && 448 copy_size > kattr->test.data_size_out) { 449 copy_size = kattr->test.data_size_out; 450 err = -ENOSPC; 451 } 452 453 if (data_out) { 454 int len = sinfo ? copy_size - sinfo->xdp_frags_size : copy_size; 455 456 if (len < 0) { 457 err = -ENOSPC; 458 goto out; 459 } 460 461 if (copy_to_user(data_out, data, len)) 462 goto out; 463 464 if (sinfo) { 465 int i, offset = len; 466 u32 data_len; 467 468 for (i = 0; i < sinfo->nr_frags; i++) { 469 skb_frag_t *frag = &sinfo->frags[i]; 470 471 if (offset >= copy_size) { 472 err = -ENOSPC; 473 break; 474 } 475 476 data_len = min_t(u32, copy_size - offset, 477 skb_frag_size(frag)); 478 479 if (copy_to_user(data_out + offset, 480 skb_frag_address(frag), 481 data_len)) 482 goto out; 483 484 offset += data_len; 485 } 486 } 487 } 488 489 if (copy_to_user(&uattr->test.data_size_out, &size, sizeof(size))) 490 goto out; 491 if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval))) 492 goto out; 493 if (copy_to_user(&uattr->test.duration, &duration, sizeof(duration))) 494 goto out; 495 if (err != -ENOSPC) 496 err = 0; 497out: 498 trace_bpf_test_finish(&err); 499 return err; 500} 501 502/* Integer types of various sizes and pointer combinations cover variety of 503 * architecture dependent calling conventions. 7+ can be supported in the 504 * future. 505 */ 506__diag_push(); 507__diag_ignore_all("-Wmissing-prototypes", 508 "Global functions as their definitions will be in vmlinux BTF"); 509__bpf_kfunc int bpf_fentry_test1(int a) 510{ 511 return a + 1; 512} 513EXPORT_SYMBOL_GPL(bpf_fentry_test1); 514 515int noinline bpf_fentry_test2(int a, u64 b) 516{ 517 return a + b; 518} 519 520int noinline bpf_fentry_test3(char a, int b, u64 c) 521{ 522 return a + b + c; 523} 524 525int noinline bpf_fentry_test4(void *a, char b, int c, u64 d) 526{ 527 return (long)a + b + c + d; 528} 529 530int noinline bpf_fentry_test5(u64 a, void *b, short c, int d, u64 e) 531{ 532 return a + (long)b + c + d + e; 533} 534 535int noinline bpf_fentry_test6(u64 a, void *b, short c, int d, void *e, u64 f) 536{ 537 return a + (long)b + c + d + (long)e + f; 538} 539 540struct bpf_fentry_test_t { 541 struct bpf_fentry_test_t *a; 542}; 543 544int noinline bpf_fentry_test7(struct bpf_fentry_test_t *arg) 545{ 546 asm volatile ("": "+r"(arg)); 547 return (long)arg; 548} 549 550int noinline bpf_fentry_test8(struct bpf_fentry_test_t *arg) 551{ 552 return (long)arg->a; 553} 554 555__bpf_kfunc u32 bpf_fentry_test9(u32 *a) 556{ 557 return *a; 558} 559 560void noinline bpf_fentry_test_sinfo(struct skb_shared_info *sinfo) 561{ 562} 563 564__bpf_kfunc int bpf_modify_return_test(int a, int *b) 565{ 566 *b += 1; 567 return a + *b; 568} 569 570__bpf_kfunc int bpf_modify_return_test2(int a, int *b, short c, int d, 571 void *e, char f, int g) 572{ 573 *b += 1; 574 return a + *b + c + d + (long)e + f + g; 575} 576 577int noinline bpf_fentry_shadow_test(int a) 578{ 579 return a + 1; 580} 581 582struct prog_test_member1 { 583 int a; 584}; 585 586struct prog_test_member { 587 struct prog_test_member1 m; 588 int c; 589}; 590 591struct prog_test_ref_kfunc { 592 int a; 593 int b; 594 struct prog_test_member memb; 595 struct prog_test_ref_kfunc *next; 596 refcount_t cnt; 597}; 598 599__bpf_kfunc void bpf_kfunc_call_test_release(struct prog_test_ref_kfunc *p) 600{ 601 refcount_dec(&p->cnt); 602} 603 604__bpf_kfunc void bpf_kfunc_call_memb_release(struct prog_test_member *p) 605{ 606} 607 608__diag_pop(); 609 610BTF_SET8_START(bpf_test_modify_return_ids) 611BTF_ID_FLAGS(func, bpf_modify_return_test) 612BTF_ID_FLAGS(func, bpf_modify_return_test2) 613BTF_ID_FLAGS(func, bpf_fentry_test1, KF_SLEEPABLE) 614BTF_SET8_END(bpf_test_modify_return_ids) 615 616static const struct btf_kfunc_id_set bpf_test_modify_return_set = { 617 .owner = THIS_MODULE, 618 .set = &bpf_test_modify_return_ids, 619}; 620 621BTF_SET8_START(test_sk_check_kfunc_ids) 622BTF_ID_FLAGS(func, bpf_kfunc_call_test_release, KF_RELEASE) 623BTF_ID_FLAGS(func, bpf_kfunc_call_memb_release, KF_RELEASE) 624BTF_SET8_END(test_sk_check_kfunc_ids) 625 626static void *bpf_test_init(const union bpf_attr *kattr, u32 user_size, 627 u32 size, u32 headroom, u32 tailroom) 628{ 629 void __user *data_in = u64_to_user_ptr(kattr->test.data_in); 630 void *data; 631 632 if (size < ETH_HLEN || size > PAGE_SIZE - headroom - tailroom) 633 return ERR_PTR(-EINVAL); 634 635 if (user_size > size) 636 return ERR_PTR(-EMSGSIZE); 637 638 size = SKB_DATA_ALIGN(size); 639 data = kzalloc(size + headroom + tailroom, GFP_USER); 640 if (!data) 641 return ERR_PTR(-ENOMEM); 642 643 if (copy_from_user(data + headroom, data_in, user_size)) { 644 kfree(data); 645 return ERR_PTR(-EFAULT); 646 } 647 648 return data; 649} 650 651int bpf_prog_test_run_tracing(struct bpf_prog *prog, 652 const union bpf_attr *kattr, 653 union bpf_attr __user *uattr) 654{ 655 struct bpf_fentry_test_t arg = {}; 656 u16 side_effect = 0, ret = 0; 657 int b = 2, err = -EFAULT; 658 u32 retval = 0; 659 660 if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size) 661 return -EINVAL; 662 663 switch (prog->expected_attach_type) { 664 case BPF_TRACE_FENTRY: 665 case BPF_TRACE_FEXIT: 666 if (bpf_fentry_test1(1) != 2 || 667 bpf_fentry_test2(2, 3) != 5 || 668 bpf_fentry_test3(4, 5, 6) != 15 || 669 bpf_fentry_test4((void *)7, 8, 9, 10) != 34 || 670 bpf_fentry_test5(11, (void *)12, 13, 14, 15) != 65 || 671 bpf_fentry_test6(16, (void *)17, 18, 19, (void *)20, 21) != 111 || 672 bpf_fentry_test7((struct bpf_fentry_test_t *)0) != 0 || 673 bpf_fentry_test8(&arg) != 0 || 674 bpf_fentry_test9(&retval) != 0) 675 goto out; 676 break; 677 case BPF_MODIFY_RETURN: 678 ret = bpf_modify_return_test(1, &b); 679 if (b != 2) 680 side_effect++; 681 b = 2; 682 ret += bpf_modify_return_test2(1, &b, 3, 4, (void *)5, 6, 7); 683 if (b != 2) 684 side_effect++; 685 break; 686 default: 687 goto out; 688 } 689 690 retval = ((u32)side_effect << 16) | ret; 691 if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval))) 692 goto out; 693 694 err = 0; 695out: 696 trace_bpf_test_finish(&err); 697 return err; 698} 699 700struct bpf_raw_tp_test_run_info { 701 struct bpf_prog *prog; 702 void *ctx; 703 u32 retval; 704}; 705 706static void 707__bpf_prog_test_run_raw_tp(void *data) 708{ 709 struct bpf_raw_tp_test_run_info *info = data; 710 711 rcu_read_lock(); 712 info->retval = bpf_prog_run(info->prog, info->ctx); 713 rcu_read_unlock(); 714} 715 716int bpf_prog_test_run_raw_tp(struct bpf_prog *prog, 717 const union bpf_attr *kattr, 718 union bpf_attr __user *uattr) 719{ 720 void __user *ctx_in = u64_to_user_ptr(kattr->test.ctx_in); 721 __u32 ctx_size_in = kattr->test.ctx_size_in; 722 struct bpf_raw_tp_test_run_info info; 723 int cpu = kattr->test.cpu, err = 0; 724 int current_cpu; 725 726 /* doesn't support data_in/out, ctx_out, duration, or repeat */ 727 if (kattr->test.data_in || kattr->test.data_out || 728 kattr->test.ctx_out || kattr->test.duration || 729 kattr->test.repeat || kattr->test.batch_size) 730 return -EINVAL; 731 732 if (ctx_size_in < prog->aux->max_ctx_offset || 733 ctx_size_in > MAX_BPF_FUNC_ARGS * sizeof(u64)) 734 return -EINVAL; 735 736 if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 && cpu != 0) 737 return -EINVAL; 738 739 if (ctx_size_in) { 740 info.ctx = memdup_user(ctx_in, ctx_size_in); 741 if (IS_ERR(info.ctx)) 742 return PTR_ERR(info.ctx); 743 } else { 744 info.ctx = NULL; 745 } 746 747 info.prog = prog; 748 749 current_cpu = get_cpu(); 750 if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 || 751 cpu == current_cpu) { 752 __bpf_prog_test_run_raw_tp(&info); 753 } else if (cpu >= nr_cpu_ids || !cpu_online(cpu)) { 754 /* smp_call_function_single() also checks cpu_online() 755 * after csd_lock(). However, since cpu is from user 756 * space, let's do an extra quick check to filter out 757 * invalid value before smp_call_function_single(). 758 */ 759 err = -ENXIO; 760 } else { 761 err = smp_call_function_single(cpu, __bpf_prog_test_run_raw_tp, 762 &info, 1); 763 } 764 put_cpu(); 765 766 if (!err && 767 copy_to_user(&uattr->test.retval, &info.retval, sizeof(u32))) 768 err = -EFAULT; 769 770 kfree(info.ctx); 771 return err; 772} 773 774static void *bpf_ctx_init(const union bpf_attr *kattr, u32 max_size) 775{ 776 void __user *data_in = u64_to_user_ptr(kattr->test.ctx_in); 777 void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out); 778 u32 size = kattr->test.ctx_size_in; 779 void *data; 780 int err; 781 782 if (!data_in && !data_out) 783 return NULL; 784 785 data = kzalloc(max_size, GFP_USER); 786 if (!data) 787 return ERR_PTR(-ENOMEM); 788 789 if (data_in) { 790 err = bpf_check_uarg_tail_zero(USER_BPFPTR(data_in), max_size, size); 791 if (err) { 792 kfree(data); 793 return ERR_PTR(err); 794 } 795 796 size = min_t(u32, max_size, size); 797 if (copy_from_user(data, data_in, size)) { 798 kfree(data); 799 return ERR_PTR(-EFAULT); 800 } 801 } 802 return data; 803} 804 805static int bpf_ctx_finish(const union bpf_attr *kattr, 806 union bpf_attr __user *uattr, const void *data, 807 u32 size) 808{ 809 void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out); 810 int err = -EFAULT; 811 u32 copy_size = size; 812 813 if (!data || !data_out) 814 return 0; 815 816 if (copy_size > kattr->test.ctx_size_out) { 817 copy_size = kattr->test.ctx_size_out; 818 err = -ENOSPC; 819 } 820 821 if (copy_to_user(data_out, data, copy_size)) 822 goto out; 823 if (copy_to_user(&uattr->test.ctx_size_out, &size, sizeof(size))) 824 goto out; 825 if (err != -ENOSPC) 826 err = 0; 827out: 828 return err; 829} 830 831/** 832 * range_is_zero - test whether buffer is initialized 833 * @buf: buffer to check 834 * @from: check from this position 835 * @to: check up until (excluding) this position 836 * 837 * This function returns true if the there is a non-zero byte 838 * in the buf in the range [from,to). 839 */ 840static inline bool range_is_zero(void *buf, size_t from, size_t to) 841{ 842 return !memchr_inv((u8 *)buf + from, 0, to - from); 843} 844 845static int convert___skb_to_skb(struct sk_buff *skb, struct __sk_buff *__skb) 846{ 847 struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb; 848 849 if (!__skb) 850 return 0; 851 852 /* make sure the fields we don't use are zeroed */ 853 if (!range_is_zero(__skb, 0, offsetof(struct __sk_buff, mark))) 854 return -EINVAL; 855 856 /* mark is allowed */ 857 858 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, mark), 859 offsetof(struct __sk_buff, priority))) 860 return -EINVAL; 861 862 /* priority is allowed */ 863 /* ingress_ifindex is allowed */ 864 /* ifindex is allowed */ 865 866 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, ifindex), 867 offsetof(struct __sk_buff, cb))) 868 return -EINVAL; 869 870 /* cb is allowed */ 871 872 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, cb), 873 offsetof(struct __sk_buff, tstamp))) 874 return -EINVAL; 875 876 /* tstamp is allowed */ 877 /* wire_len is allowed */ 878 /* gso_segs is allowed */ 879 880 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, gso_segs), 881 offsetof(struct __sk_buff, gso_size))) 882 return -EINVAL; 883 884 /* gso_size is allowed */ 885 886 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, gso_size), 887 offsetof(struct __sk_buff, hwtstamp))) 888 return -EINVAL; 889 890 /* hwtstamp is allowed */ 891 892 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, hwtstamp), 893 sizeof(struct __sk_buff))) 894 return -EINVAL; 895 896 skb->mark = __skb->mark; 897 skb->priority = __skb->priority; 898 skb->skb_iif = __skb->ingress_ifindex; 899 skb->tstamp = __skb->tstamp; 900 memcpy(&cb->data, __skb->cb, QDISC_CB_PRIV_LEN); 901 902 if (__skb->wire_len == 0) { 903 cb->pkt_len = skb->len; 904 } else { 905 if (__skb->wire_len < skb->len || 906 __skb->wire_len > GSO_LEGACY_MAX_SIZE) 907 return -EINVAL; 908 cb->pkt_len = __skb->wire_len; 909 } 910 911 if (__skb->gso_segs > GSO_MAX_SEGS) 912 return -EINVAL; 913 skb_shinfo(skb)->gso_segs = __skb->gso_segs; 914 skb_shinfo(skb)->gso_size = __skb->gso_size; 915 skb_shinfo(skb)->hwtstamps.hwtstamp = __skb->hwtstamp; 916 917 return 0; 918} 919 920static void convert_skb_to___skb(struct sk_buff *skb, struct __sk_buff *__skb) 921{ 922 struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb; 923 924 if (!__skb) 925 return; 926 927 __skb->mark = skb->mark; 928 __skb->priority = skb->priority; 929 __skb->ingress_ifindex = skb->skb_iif; 930 __skb->ifindex = skb->dev->ifindex; 931 __skb->tstamp = skb->tstamp; 932 memcpy(__skb->cb, &cb->data, QDISC_CB_PRIV_LEN); 933 __skb->wire_len = cb->pkt_len; 934 __skb->gso_segs = skb_shinfo(skb)->gso_segs; 935 __skb->hwtstamp = skb_shinfo(skb)->hwtstamps.hwtstamp; 936} 937 938static struct proto bpf_dummy_proto = { 939 .name = "bpf_dummy", 940 .owner = THIS_MODULE, 941 .obj_size = sizeof(struct sock), 942}; 943 944int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr, 945 union bpf_attr __user *uattr) 946{ 947 bool is_l2 = false, is_direct_pkt_access = false; 948 struct net *net = current->nsproxy->net_ns; 949 struct net_device *dev = net->loopback_dev; 950 u32 size = kattr->test.data_size_in; 951 u32 repeat = kattr->test.repeat; 952 struct __sk_buff *ctx = NULL; 953 u32 retval, duration; 954 int hh_len = ETH_HLEN; 955 struct sk_buff *skb; 956 struct sock *sk; 957 void *data; 958 int ret; 959 960 if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size) 961 return -EINVAL; 962 963 data = bpf_test_init(kattr, kattr->test.data_size_in, 964 size, NET_SKB_PAD + NET_IP_ALIGN, 965 SKB_DATA_ALIGN(sizeof(struct skb_shared_info))); 966 if (IS_ERR(data)) 967 return PTR_ERR(data); 968 969 ctx = bpf_ctx_init(kattr, sizeof(struct __sk_buff)); 970 if (IS_ERR(ctx)) { 971 kfree(data); 972 return PTR_ERR(ctx); 973 } 974 975 switch (prog->type) { 976 case BPF_PROG_TYPE_SCHED_CLS: 977 case BPF_PROG_TYPE_SCHED_ACT: 978 is_l2 = true; 979 fallthrough; 980 case BPF_PROG_TYPE_LWT_IN: 981 case BPF_PROG_TYPE_LWT_OUT: 982 case BPF_PROG_TYPE_LWT_XMIT: 983 is_direct_pkt_access = true; 984 break; 985 default: 986 break; 987 } 988 989 sk = sk_alloc(net, AF_UNSPEC, GFP_USER, &bpf_dummy_proto, 1); 990 if (!sk) { 991 kfree(data); 992 kfree(ctx); 993 return -ENOMEM; 994 } 995 sock_init_data(NULL, sk); 996 997 skb = slab_build_skb(data); 998 if (!skb) { 999 kfree(data); 1000 kfree(ctx); 1001 sk_free(sk); 1002 return -ENOMEM; 1003 } 1004 skb->sk = sk; 1005 1006 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN); 1007 __skb_put(skb, size); 1008 if (ctx && ctx->ifindex > 1) { 1009 dev = dev_get_by_index(net, ctx->ifindex); 1010 if (!dev) { 1011 ret = -ENODEV; 1012 goto out; 1013 } 1014 } 1015 skb->protocol = eth_type_trans(skb, dev); 1016 skb_reset_network_header(skb); 1017 1018 switch (skb->protocol) { 1019 case htons(ETH_P_IP): 1020 sk->sk_family = AF_INET; 1021 if (sizeof(struct iphdr) <= skb_headlen(skb)) { 1022 sk->sk_rcv_saddr = ip_hdr(skb)->saddr; 1023 sk->sk_daddr = ip_hdr(skb)->daddr; 1024 } 1025 break; 1026#if IS_ENABLED(CONFIG_IPV6) 1027 case htons(ETH_P_IPV6): 1028 sk->sk_family = AF_INET6; 1029 if (sizeof(struct ipv6hdr) <= skb_headlen(skb)) { 1030 sk->sk_v6_rcv_saddr = ipv6_hdr(skb)->saddr; 1031 sk->sk_v6_daddr = ipv6_hdr(skb)->daddr; 1032 } 1033 break; 1034#endif 1035 default: 1036 break; 1037 } 1038 1039 if (is_l2) 1040 __skb_push(skb, hh_len); 1041 if (is_direct_pkt_access) 1042 bpf_compute_data_pointers(skb); 1043 ret = convert___skb_to_skb(skb, ctx); 1044 if (ret) 1045 goto out; 1046 ret = bpf_test_run(prog, skb, repeat, &retval, &duration, false); 1047 if (ret) 1048 goto out; 1049 if (!is_l2) { 1050 if (skb_headroom(skb) < hh_len) { 1051 int nhead = HH_DATA_ALIGN(hh_len - skb_headroom(skb)); 1052 1053 if (pskb_expand_head(skb, nhead, 0, GFP_USER)) { 1054 ret = -ENOMEM; 1055 goto out; 1056 } 1057 } 1058 memset(__skb_push(skb, hh_len), 0, hh_len); 1059 } 1060 convert_skb_to___skb(skb, ctx); 1061 1062 size = skb->len; 1063 /* bpf program can never convert linear skb to non-linear */ 1064 if (WARN_ON_ONCE(skb_is_nonlinear(skb))) 1065 size = skb_headlen(skb); 1066 ret = bpf_test_finish(kattr, uattr, skb->data, NULL, size, retval, 1067 duration); 1068 if (!ret) 1069 ret = bpf_ctx_finish(kattr, uattr, ctx, 1070 sizeof(struct __sk_buff)); 1071out: 1072 if (dev && dev != net->loopback_dev) 1073 dev_put(dev); 1074 kfree_skb(skb); 1075 sk_free(sk); 1076 kfree(ctx); 1077 return ret; 1078} 1079 1080static int xdp_convert_md_to_buff(struct xdp_md *xdp_md, struct xdp_buff *xdp) 1081{ 1082 unsigned int ingress_ifindex, rx_queue_index; 1083 struct netdev_rx_queue *rxqueue; 1084 struct net_device *device; 1085 1086 if (!xdp_md) 1087 return 0; 1088 1089 if (xdp_md->egress_ifindex != 0) 1090 return -EINVAL; 1091 1092 ingress_ifindex = xdp_md->ingress_ifindex; 1093 rx_queue_index = xdp_md->rx_queue_index; 1094 1095 if (!ingress_ifindex && rx_queue_index) 1096 return -EINVAL; 1097 1098 if (ingress_ifindex) { 1099 device = dev_get_by_index(current->nsproxy->net_ns, 1100 ingress_ifindex); 1101 if (!device) 1102 return -ENODEV; 1103 1104 if (rx_queue_index >= device->real_num_rx_queues) 1105 goto free_dev; 1106 1107 rxqueue = __netif_get_rx_queue(device, rx_queue_index); 1108 1109 if (!xdp_rxq_info_is_reg(&rxqueue->xdp_rxq)) 1110 goto free_dev; 1111 1112 xdp->rxq = &rxqueue->xdp_rxq; 1113 /* The device is now tracked in the xdp->rxq for later 1114 * dev_put() 1115 */ 1116 } 1117 1118 xdp->data = xdp->data_meta + xdp_md->data; 1119 return 0; 1120 1121free_dev: 1122 dev_put(device); 1123 return -EINVAL; 1124} 1125 1126static void xdp_convert_buff_to_md(struct xdp_buff *xdp, struct xdp_md *xdp_md) 1127{ 1128 if (!xdp_md) 1129 return; 1130 1131 xdp_md->data = xdp->data - xdp->data_meta; 1132 xdp_md->data_end = xdp->data_end - xdp->data_meta; 1133 1134 if (xdp_md->ingress_ifindex) 1135 dev_put(xdp->rxq->dev); 1136} 1137 1138int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr, 1139 union bpf_attr __user *uattr) 1140{ 1141 bool do_live = (kattr->test.flags & BPF_F_TEST_XDP_LIVE_FRAMES); 1142 u32 tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 1143 u32 batch_size = kattr->test.batch_size; 1144 u32 retval = 0, duration, max_data_sz; 1145 u32 size = kattr->test.data_size_in; 1146 u32 headroom = XDP_PACKET_HEADROOM; 1147 u32 repeat = kattr->test.repeat; 1148 struct netdev_rx_queue *rxqueue; 1149 struct skb_shared_info *sinfo; 1150 struct xdp_buff xdp = {}; 1151 int i, ret = -EINVAL; 1152 struct xdp_md *ctx; 1153 void *data; 1154 1155 if (prog->expected_attach_type == BPF_XDP_DEVMAP || 1156 prog->expected_attach_type == BPF_XDP_CPUMAP) 1157 return -EINVAL; 1158 1159 if (kattr->test.flags & ~BPF_F_TEST_XDP_LIVE_FRAMES) 1160 return -EINVAL; 1161 1162 if (bpf_prog_is_dev_bound(prog->aux)) 1163 return -EINVAL; 1164 1165 if (do_live) { 1166 if (!batch_size) 1167 batch_size = NAPI_POLL_WEIGHT; 1168 else if (batch_size > TEST_XDP_MAX_BATCH) 1169 return -E2BIG; 1170 1171 headroom += sizeof(struct xdp_page_head); 1172 } else if (batch_size) { 1173 return -EINVAL; 1174 } 1175 1176 ctx = bpf_ctx_init(kattr, sizeof(struct xdp_md)); 1177 if (IS_ERR(ctx)) 1178 return PTR_ERR(ctx); 1179 1180 if (ctx) { 1181 /* There can't be user provided data before the meta data */ 1182 if (ctx->data_meta || ctx->data_end != size || 1183 ctx->data > ctx->data_end || 1184 unlikely(xdp_metalen_invalid(ctx->data)) || 1185 (do_live && (kattr->test.data_out || kattr->test.ctx_out))) 1186 goto free_ctx; 1187 /* Meta data is allocated from the headroom */ 1188 headroom -= ctx->data; 1189 } 1190 1191 max_data_sz = 4096 - headroom - tailroom; 1192 if (size > max_data_sz) { 1193 /* disallow live data mode for jumbo frames */ 1194 if (do_live) 1195 goto free_ctx; 1196 size = max_data_sz; 1197 } 1198 1199 data = bpf_test_init(kattr, size, max_data_sz, headroom, tailroom); 1200 if (IS_ERR(data)) { 1201 ret = PTR_ERR(data); 1202 goto free_ctx; 1203 } 1204 1205 rxqueue = __netif_get_rx_queue(current->nsproxy->net_ns->loopback_dev, 0); 1206 rxqueue->xdp_rxq.frag_size = headroom + max_data_sz + tailroom; 1207 xdp_init_buff(&xdp, rxqueue->xdp_rxq.frag_size, &rxqueue->xdp_rxq); 1208 xdp_prepare_buff(&xdp, data, headroom, size, true); 1209 sinfo = xdp_get_shared_info_from_buff(&xdp); 1210 1211 ret = xdp_convert_md_to_buff(ctx, &xdp); 1212 if (ret) 1213 goto free_data; 1214 1215 if (unlikely(kattr->test.data_size_in > size)) { 1216 void __user *data_in = u64_to_user_ptr(kattr->test.data_in); 1217 1218 while (size < kattr->test.data_size_in) { 1219 struct page *page; 1220 skb_frag_t *frag; 1221 u32 data_len; 1222 1223 if (sinfo->nr_frags == MAX_SKB_FRAGS) { 1224 ret = -ENOMEM; 1225 goto out; 1226 } 1227 1228 page = alloc_page(GFP_KERNEL); 1229 if (!page) { 1230 ret = -ENOMEM; 1231 goto out; 1232 } 1233 1234 frag = &sinfo->frags[sinfo->nr_frags++]; 1235 1236 data_len = min_t(u32, kattr->test.data_size_in - size, 1237 PAGE_SIZE); 1238 skb_frag_fill_page_desc(frag, page, 0, data_len); 1239 1240 if (copy_from_user(page_address(page), data_in + size, 1241 data_len)) { 1242 ret = -EFAULT; 1243 goto out; 1244 } 1245 sinfo->xdp_frags_size += data_len; 1246 size += data_len; 1247 } 1248 xdp_buff_set_frags_flag(&xdp); 1249 } 1250 1251 if (repeat > 1) 1252 bpf_prog_change_xdp(NULL, prog); 1253 1254 if (do_live) 1255 ret = bpf_test_run_xdp_live(prog, &xdp, repeat, batch_size, &duration); 1256 else 1257 ret = bpf_test_run(prog, &xdp, repeat, &retval, &duration, true); 1258 /* We convert the xdp_buff back to an xdp_md before checking the return 1259 * code so the reference count of any held netdevice will be decremented 1260 * even if the test run failed. 1261 */ 1262 xdp_convert_buff_to_md(&xdp, ctx); 1263 if (ret) 1264 goto out; 1265 1266 size = xdp.data_end - xdp.data_meta + sinfo->xdp_frags_size; 1267 ret = bpf_test_finish(kattr, uattr, xdp.data_meta, sinfo, size, 1268 retval, duration); 1269 if (!ret) 1270 ret = bpf_ctx_finish(kattr, uattr, ctx, 1271 sizeof(struct xdp_md)); 1272 1273out: 1274 if (repeat > 1) 1275 bpf_prog_change_xdp(prog, NULL); 1276free_data: 1277 for (i = 0; i < sinfo->nr_frags; i++) 1278 __free_page(skb_frag_page(&sinfo->frags[i])); 1279 kfree(data); 1280free_ctx: 1281 kfree(ctx); 1282 return ret; 1283} 1284 1285static int verify_user_bpf_flow_keys(struct bpf_flow_keys *ctx) 1286{ 1287 /* make sure the fields we don't use are zeroed */ 1288 if (!range_is_zero(ctx, 0, offsetof(struct bpf_flow_keys, flags))) 1289 return -EINVAL; 1290 1291 /* flags is allowed */ 1292 1293 if (!range_is_zero(ctx, offsetofend(struct bpf_flow_keys, flags), 1294 sizeof(struct bpf_flow_keys))) 1295 return -EINVAL; 1296 1297 return 0; 1298} 1299 1300int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog, 1301 const union bpf_attr *kattr, 1302 union bpf_attr __user *uattr) 1303{ 1304 struct bpf_test_timer t = { NO_PREEMPT }; 1305 u32 size = kattr->test.data_size_in; 1306 struct bpf_flow_dissector ctx = {}; 1307 u32 repeat = kattr->test.repeat; 1308 struct bpf_flow_keys *user_ctx; 1309 struct bpf_flow_keys flow_keys; 1310 const struct ethhdr *eth; 1311 unsigned int flags = 0; 1312 u32 retval, duration; 1313 void *data; 1314 int ret; 1315 1316 if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size) 1317 return -EINVAL; 1318 1319 if (size < ETH_HLEN) 1320 return -EINVAL; 1321 1322 data = bpf_test_init(kattr, kattr->test.data_size_in, size, 0, 0); 1323 if (IS_ERR(data)) 1324 return PTR_ERR(data); 1325 1326 eth = (struct ethhdr *)data; 1327 1328 if (!repeat) 1329 repeat = 1; 1330 1331 user_ctx = bpf_ctx_init(kattr, sizeof(struct bpf_flow_keys)); 1332 if (IS_ERR(user_ctx)) { 1333 kfree(data); 1334 return PTR_ERR(user_ctx); 1335 } 1336 if (user_ctx) { 1337 ret = verify_user_bpf_flow_keys(user_ctx); 1338 if (ret) 1339 goto out; 1340 flags = user_ctx->flags; 1341 } 1342 1343 ctx.flow_keys = &flow_keys; 1344 ctx.data = data; 1345 ctx.data_end = (__u8 *)data + size; 1346 1347 bpf_test_timer_enter(&t); 1348 do { 1349 retval = bpf_flow_dissect(prog, &ctx, eth->h_proto, ETH_HLEN, 1350 size, flags); 1351 } while (bpf_test_timer_continue(&t, 1, repeat, &ret, &duration)); 1352 bpf_test_timer_leave(&t); 1353 1354 if (ret < 0) 1355 goto out; 1356 1357 ret = bpf_test_finish(kattr, uattr, &flow_keys, NULL, 1358 sizeof(flow_keys), retval, duration); 1359 if (!ret) 1360 ret = bpf_ctx_finish(kattr, uattr, user_ctx, 1361 sizeof(struct bpf_flow_keys)); 1362 1363out: 1364 kfree(user_ctx); 1365 kfree(data); 1366 return ret; 1367} 1368 1369int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog, const union bpf_attr *kattr, 1370 union bpf_attr __user *uattr) 1371{ 1372 struct bpf_test_timer t = { NO_PREEMPT }; 1373 struct bpf_prog_array *progs = NULL; 1374 struct bpf_sk_lookup_kern ctx = {}; 1375 u32 repeat = kattr->test.repeat; 1376 struct bpf_sk_lookup *user_ctx; 1377 u32 retval, duration; 1378 int ret = -EINVAL; 1379 1380 if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size) 1381 return -EINVAL; 1382 1383 if (kattr->test.data_in || kattr->test.data_size_in || kattr->test.data_out || 1384 kattr->test.data_size_out) 1385 return -EINVAL; 1386 1387 if (!repeat) 1388 repeat = 1; 1389 1390 user_ctx = bpf_ctx_init(kattr, sizeof(*user_ctx)); 1391 if (IS_ERR(user_ctx)) 1392 return PTR_ERR(user_ctx); 1393 1394 if (!user_ctx) 1395 return -EINVAL; 1396 1397 if (user_ctx->sk) 1398 goto out; 1399 1400 if (!range_is_zero(user_ctx, offsetofend(typeof(*user_ctx), local_port), sizeof(*user_ctx))) 1401 goto out; 1402 1403 if (user_ctx->local_port > U16_MAX) { 1404 ret = -ERANGE; 1405 goto out; 1406 } 1407 1408 ctx.family = (u16)user_ctx->family; 1409 ctx.protocol = (u16)user_ctx->protocol; 1410 ctx.dport = (u16)user_ctx->local_port; 1411 ctx.sport = user_ctx->remote_port; 1412 1413 switch (ctx.family) { 1414 case AF_INET: 1415 ctx.v4.daddr = (__force __be32)user_ctx->local_ip4; 1416 ctx.v4.saddr = (__force __be32)user_ctx->remote_ip4; 1417 break; 1418 1419#if IS_ENABLED(CONFIG_IPV6) 1420 case AF_INET6: 1421 ctx.v6.daddr = (struct in6_addr *)user_ctx->local_ip6; 1422 ctx.v6.saddr = (struct in6_addr *)user_ctx->remote_ip6; 1423 break; 1424#endif 1425 1426 default: 1427 ret = -EAFNOSUPPORT; 1428 goto out; 1429 } 1430 1431 progs = bpf_prog_array_alloc(1, GFP_KERNEL); 1432 if (!progs) { 1433 ret = -ENOMEM; 1434 goto out; 1435 } 1436 1437 progs->items[0].prog = prog; 1438 1439 bpf_test_timer_enter(&t); 1440 do { 1441 ctx.selected_sk = NULL; 1442 retval = BPF_PROG_SK_LOOKUP_RUN_ARRAY(progs, ctx, bpf_prog_run); 1443 } while (bpf_test_timer_continue(&t, 1, repeat, &ret, &duration)); 1444 bpf_test_timer_leave(&t); 1445 1446 if (ret < 0) 1447 goto out; 1448 1449 user_ctx->cookie = 0; 1450 if (ctx.selected_sk) { 1451 if (ctx.selected_sk->sk_reuseport && !ctx.no_reuseport) { 1452 ret = -EOPNOTSUPP; 1453 goto out; 1454 } 1455 1456 user_ctx->cookie = sock_gen_cookie(ctx.selected_sk); 1457 } 1458 1459 ret = bpf_test_finish(kattr, uattr, NULL, NULL, 0, retval, duration); 1460 if (!ret) 1461 ret = bpf_ctx_finish(kattr, uattr, user_ctx, sizeof(*user_ctx)); 1462 1463out: 1464 bpf_prog_array_free(progs); 1465 kfree(user_ctx); 1466 return ret; 1467} 1468 1469int bpf_prog_test_run_syscall(struct bpf_prog *prog, 1470 const union bpf_attr *kattr, 1471 union bpf_attr __user *uattr) 1472{ 1473 void __user *ctx_in = u64_to_user_ptr(kattr->test.ctx_in); 1474 __u32 ctx_size_in = kattr->test.ctx_size_in; 1475 void *ctx = NULL; 1476 u32 retval; 1477 int err = 0; 1478 1479 /* doesn't support data_in/out, ctx_out, duration, or repeat or flags */ 1480 if (kattr->test.data_in || kattr->test.data_out || 1481 kattr->test.ctx_out || kattr->test.duration || 1482 kattr->test.repeat || kattr->test.flags || 1483 kattr->test.batch_size) 1484 return -EINVAL; 1485 1486 if (ctx_size_in < prog->aux->max_ctx_offset || 1487 ctx_size_in > U16_MAX) 1488 return -EINVAL; 1489 1490 if (ctx_size_in) { 1491 ctx = memdup_user(ctx_in, ctx_size_in); 1492 if (IS_ERR(ctx)) 1493 return PTR_ERR(ctx); 1494 } 1495 1496 rcu_read_lock_trace(); 1497 retval = bpf_prog_run_pin_on_cpu(prog, ctx); 1498 rcu_read_unlock_trace(); 1499 1500 if (copy_to_user(&uattr->test.retval, &retval, sizeof(u32))) { 1501 err = -EFAULT; 1502 goto out; 1503 } 1504 if (ctx_size_in) 1505 if (copy_to_user(ctx_in, ctx, ctx_size_in)) 1506 err = -EFAULT; 1507out: 1508 kfree(ctx); 1509 return err; 1510} 1511 1512static int verify_and_copy_hook_state(struct nf_hook_state *state, 1513 const struct nf_hook_state *user, 1514 struct net_device *dev) 1515{ 1516 if (user->in || user->out) 1517 return -EINVAL; 1518 1519 if (user->net || user->sk || user->okfn) 1520 return -EINVAL; 1521 1522 switch (user->pf) { 1523 case NFPROTO_IPV4: 1524 case NFPROTO_IPV6: 1525 switch (state->hook) { 1526 case NF_INET_PRE_ROUTING: 1527 state->in = dev; 1528 break; 1529 case NF_INET_LOCAL_IN: 1530 state->in = dev; 1531 break; 1532 case NF_INET_FORWARD: 1533 state->in = dev; 1534 state->out = dev; 1535 break; 1536 case NF_INET_LOCAL_OUT: 1537 state->out = dev; 1538 break; 1539 case NF_INET_POST_ROUTING: 1540 state->out = dev; 1541 break; 1542 } 1543 1544 break; 1545 default: 1546 return -EINVAL; 1547 } 1548 1549 state->pf = user->pf; 1550 state->hook = user->hook; 1551 1552 return 0; 1553} 1554 1555static __be16 nfproto_eth(int nfproto) 1556{ 1557 switch (nfproto) { 1558 case NFPROTO_IPV4: 1559 return htons(ETH_P_IP); 1560 case NFPROTO_IPV6: 1561 break; 1562 } 1563 1564 return htons(ETH_P_IPV6); 1565} 1566 1567int bpf_prog_test_run_nf(struct bpf_prog *prog, 1568 const union bpf_attr *kattr, 1569 union bpf_attr __user *uattr) 1570{ 1571 struct net *net = current->nsproxy->net_ns; 1572 struct net_device *dev = net->loopback_dev; 1573 struct nf_hook_state *user_ctx, hook_state = { 1574 .pf = NFPROTO_IPV4, 1575 .hook = NF_INET_LOCAL_OUT, 1576 }; 1577 u32 size = kattr->test.data_size_in; 1578 u32 repeat = kattr->test.repeat; 1579 struct bpf_nf_ctx ctx = { 1580 .state = &hook_state, 1581 }; 1582 struct sk_buff *skb = NULL; 1583 u32 retval, duration; 1584 void *data; 1585 int ret; 1586 1587 if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size) 1588 return -EINVAL; 1589 1590 if (size < sizeof(struct iphdr)) 1591 return -EINVAL; 1592 1593 data = bpf_test_init(kattr, kattr->test.data_size_in, size, 1594 NET_SKB_PAD + NET_IP_ALIGN, 1595 SKB_DATA_ALIGN(sizeof(struct skb_shared_info))); 1596 if (IS_ERR(data)) 1597 return PTR_ERR(data); 1598 1599 if (!repeat) 1600 repeat = 1; 1601 1602 user_ctx = bpf_ctx_init(kattr, sizeof(struct nf_hook_state)); 1603 if (IS_ERR(user_ctx)) { 1604 kfree(data); 1605 return PTR_ERR(user_ctx); 1606 } 1607 1608 if (user_ctx) { 1609 ret = verify_and_copy_hook_state(&hook_state, user_ctx, dev); 1610 if (ret) 1611 goto out; 1612 } 1613 1614 skb = slab_build_skb(data); 1615 if (!skb) { 1616 ret = -ENOMEM; 1617 goto out; 1618 } 1619 1620 data = NULL; /* data released via kfree_skb */ 1621 1622 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN); 1623 __skb_put(skb, size); 1624 1625 ret = -EINVAL; 1626 1627 if (hook_state.hook != NF_INET_LOCAL_OUT) { 1628 if (size < ETH_HLEN + sizeof(struct iphdr)) 1629 goto out; 1630 1631 skb->protocol = eth_type_trans(skb, dev); 1632 switch (skb->protocol) { 1633 case htons(ETH_P_IP): 1634 if (hook_state.pf == NFPROTO_IPV4) 1635 break; 1636 goto out; 1637 case htons(ETH_P_IPV6): 1638 if (size < ETH_HLEN + sizeof(struct ipv6hdr)) 1639 goto out; 1640 if (hook_state.pf == NFPROTO_IPV6) 1641 break; 1642 goto out; 1643 default: 1644 ret = -EPROTO; 1645 goto out; 1646 } 1647 1648 skb_reset_network_header(skb); 1649 } else { 1650 skb->protocol = nfproto_eth(hook_state.pf); 1651 } 1652 1653 ctx.skb = skb; 1654 1655 ret = bpf_test_run(prog, &ctx, repeat, &retval, &duration, false); 1656 if (ret) 1657 goto out; 1658 1659 ret = bpf_test_finish(kattr, uattr, NULL, NULL, 0, retval, duration); 1660 1661out: 1662 kfree(user_ctx); 1663 kfree_skb(skb); 1664 kfree(data); 1665 return ret; 1666} 1667 1668static const struct btf_kfunc_id_set bpf_prog_test_kfunc_set = { 1669 .owner = THIS_MODULE, 1670 .set = &test_sk_check_kfunc_ids, 1671}; 1672 1673BTF_ID_LIST(bpf_prog_test_dtor_kfunc_ids) 1674BTF_ID(struct, prog_test_ref_kfunc) 1675BTF_ID(func, bpf_kfunc_call_test_release) 1676BTF_ID(struct, prog_test_member) 1677BTF_ID(func, bpf_kfunc_call_memb_release) 1678 1679static int __init bpf_prog_test_run_init(void) 1680{ 1681 const struct btf_id_dtor_kfunc bpf_prog_test_dtor_kfunc[] = { 1682 { 1683 .btf_id = bpf_prog_test_dtor_kfunc_ids[0], 1684 .kfunc_btf_id = bpf_prog_test_dtor_kfunc_ids[1] 1685 }, 1686 { 1687 .btf_id = bpf_prog_test_dtor_kfunc_ids[2], 1688 .kfunc_btf_id = bpf_prog_test_dtor_kfunc_ids[3], 1689 }, 1690 }; 1691 int ret; 1692 1693 ret = register_btf_fmodret_id_set(&bpf_test_modify_return_set); 1694 ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SCHED_CLS, &bpf_prog_test_kfunc_set); 1695 ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, &bpf_prog_test_kfunc_set); 1696 ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, &bpf_prog_test_kfunc_set); 1697 return ret ?: register_btf_id_dtor_kfuncs(bpf_prog_test_dtor_kfunc, 1698 ARRAY_SIZE(bpf_prog_test_dtor_kfunc), 1699 THIS_MODULE); 1700} 1701late_initcall(bpf_prog_test_run_init); 1702