1// SPDX-License-Identifier: GPL-2.0 2/* Multipath TCP 3 * 4 * Copyright (c) 2017 - 2019, Intel Corporation. 5 */ 6 7#define pr_fmt(fmt) "MPTCP: " fmt 8 9#include <linux/kernel.h> 10#include <linux/module.h> 11#include <linux/netdevice.h> 12#include <crypto/algapi.h> 13#include <crypto/sha.h> 14#include <net/sock.h> 15#include <net/inet_common.h> 16#include <net/inet_hashtables.h> 17#include <net/protocol.h> 18#include <net/tcp.h> 19#if IS_ENABLED(CONFIG_MPTCP_IPV6) 20#include <net/ip6_route.h> 21#endif 22#include <net/mptcp.h> 23#include <uapi/linux/mptcp.h> 24#include "protocol.h" 25#include "mib.h" 26 27static void SUBFLOW_REQ_INC_STATS(struct request_sock *req, 28 enum linux_mptcp_mib_field field) 29{ 30 MPTCP_INC_STATS(sock_net(req_to_sk(req)), field); 31} 32 33static void subflow_req_destructor(struct request_sock *req) 34{ 35 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); 36 37 pr_debug("subflow_req=%p", subflow_req); 38 39 if (subflow_req->msk) 40 sock_put((struct sock *)subflow_req->msk); 41 42 mptcp_token_destroy_request(req); 43} 44 45static void subflow_generate_hmac(u64 key1, u64 key2, u32 nonce1, u32 nonce2, 46 void *hmac) 47{ 48 u8 msg[8]; 49 50 put_unaligned_be32(nonce1, &msg[0]); 51 put_unaligned_be32(nonce2, &msg[4]); 52 53 mptcp_crypto_hmac_sha(key1, key2, msg, 8, hmac); 54} 55 56static bool mptcp_can_accept_new_subflow(const struct mptcp_sock *msk) 57{ 58 return mptcp_is_fully_established((void *)msk) && 59 READ_ONCE(msk->pm.accept_subflow); 60} 61 62/* validate received token and create truncated hmac and nonce for SYN-ACK */ 63static struct mptcp_sock *subflow_token_join_request(struct request_sock *req, 64 const struct sk_buff *skb) 65{ 66 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); 67 u8 hmac[SHA256_DIGEST_SIZE]; 68 struct mptcp_sock *msk; 69 int local_id; 70 71 msk = mptcp_token_get_sock(sock_net(req_to_sk(req)), subflow_req->token); 72 if (!msk) { 73 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINNOTOKEN); 74 return NULL; 75 } 76 77 local_id = mptcp_pm_get_local_id(msk, (struct sock_common *)req); 78 if (local_id < 0) { 79 sock_put((struct sock *)msk); 80 return NULL; 81 } 82 subflow_req->local_id = local_id; 83 84 get_random_bytes(&subflow_req->local_nonce, sizeof(u32)); 85 86 subflow_generate_hmac(msk->local_key, msk->remote_key, 87 subflow_req->local_nonce, 88 subflow_req->remote_nonce, hmac); 89 90 subflow_req->thmac = get_unaligned_be64(hmac); 91 return msk; 92} 93 94static int __subflow_init_req(struct request_sock *req, const struct sock *sk_listener) 95{ 96 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); 97 98 subflow_req->mp_capable = 0; 99 subflow_req->mp_join = 0; 100 subflow_req->msk = NULL; 101 mptcp_token_init_request(req); 102 103#ifdef CONFIG_TCP_MD5SIG 104 /* no MPTCP if MD5SIG is enabled on this socket or we may run out of 105 * TCP option space. 106 */ 107 if (rcu_access_pointer(tcp_sk(sk_listener)->md5sig_info)) 108 return -EINVAL; 109#endif 110 111 return 0; 112} 113 114static void subflow_init_req(struct request_sock *req, 115 const struct sock *sk_listener, 116 struct sk_buff *skb) 117{ 118 struct mptcp_subflow_context *listener = mptcp_subflow_ctx(sk_listener); 119 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); 120 struct mptcp_options_received mp_opt; 121 int ret; 122 123 pr_debug("subflow_req=%p, listener=%p", subflow_req, listener); 124 125 ret = __subflow_init_req(req, sk_listener); 126 if (ret) 127 return; 128 129 mptcp_get_options(skb, &mp_opt); 130 131 if (mp_opt.mp_capable) { 132 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_MPCAPABLEPASSIVE); 133 134 if (mp_opt.mp_join) 135 return; 136 } else if (mp_opt.mp_join) { 137 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINSYNRX); 138 } 139 140 if (mp_opt.mp_capable && listener->request_mptcp) { 141 int err, retries = 4; 142 143 subflow_req->ssn_offset = TCP_SKB_CB(skb)->seq; 144again: 145 do { 146 get_random_bytes(&subflow_req->local_key, sizeof(subflow_req->local_key)); 147 } while (subflow_req->local_key == 0); 148 149 if (unlikely(req->syncookie)) { 150 mptcp_crypto_key_sha(subflow_req->local_key, 151 &subflow_req->token, 152 &subflow_req->idsn); 153 if (mptcp_token_exists(subflow_req->token)) { 154 if (retries-- > 0) 155 goto again; 156 } else { 157 subflow_req->mp_capable = 1; 158 } 159 return; 160 } 161 162 err = mptcp_token_new_request(req); 163 if (err == 0) 164 subflow_req->mp_capable = 1; 165 else if (retries-- > 0) 166 goto again; 167 168 } else if (mp_opt.mp_join && listener->request_mptcp) { 169 subflow_req->ssn_offset = TCP_SKB_CB(skb)->seq; 170 subflow_req->mp_join = 1; 171 subflow_req->backup = mp_opt.backup; 172 subflow_req->remote_id = mp_opt.join_id; 173 subflow_req->token = mp_opt.token; 174 subflow_req->remote_nonce = mp_opt.nonce; 175 subflow_req->msk = subflow_token_join_request(req, skb); 176 177 if (unlikely(req->syncookie) && subflow_req->msk) { 178 if (mptcp_can_accept_new_subflow(subflow_req->msk)) 179 subflow_init_req_cookie_join_save(subflow_req, skb); 180 } 181 182 pr_debug("token=%u, remote_nonce=%u msk=%p", subflow_req->token, 183 subflow_req->remote_nonce, subflow_req->msk); 184 } 185} 186 187int mptcp_subflow_init_cookie_req(struct request_sock *req, 188 const struct sock *sk_listener, 189 struct sk_buff *skb) 190{ 191 struct mptcp_subflow_context *listener = mptcp_subflow_ctx(sk_listener); 192 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); 193 struct mptcp_options_received mp_opt; 194 int err; 195 196 err = __subflow_init_req(req, sk_listener); 197 if (err) 198 return err; 199 200 mptcp_get_options(skb, &mp_opt); 201 202 if (mp_opt.mp_capable && mp_opt.mp_join) 203 return -EINVAL; 204 205 if (mp_opt.mp_capable && listener->request_mptcp) { 206 if (mp_opt.sndr_key == 0) 207 return -EINVAL; 208 209 subflow_req->local_key = mp_opt.rcvr_key; 210 err = mptcp_token_new_request(req); 211 if (err) 212 return err; 213 214 subflow_req->mp_capable = 1; 215 subflow_req->ssn_offset = TCP_SKB_CB(skb)->seq - 1; 216 } else if (mp_opt.mp_join && listener->request_mptcp) { 217 if (!mptcp_token_join_cookie_init_state(subflow_req, skb)) 218 return -EINVAL; 219 220 if (mptcp_can_accept_new_subflow(subflow_req->msk)) 221 subflow_req->mp_join = 1; 222 223 subflow_req->ssn_offset = TCP_SKB_CB(skb)->seq - 1; 224 } 225 226 return 0; 227} 228EXPORT_SYMBOL_GPL(mptcp_subflow_init_cookie_req); 229 230static void subflow_v4_init_req(struct request_sock *req, 231 const struct sock *sk_listener, 232 struct sk_buff *skb) 233{ 234 tcp_rsk(req)->is_mptcp = 1; 235 236 tcp_request_sock_ipv4_ops.init_req(req, sk_listener, skb); 237 238 subflow_init_req(req, sk_listener, skb); 239} 240 241#if IS_ENABLED(CONFIG_MPTCP_IPV6) 242static void subflow_v6_init_req(struct request_sock *req, 243 const struct sock *sk_listener, 244 struct sk_buff *skb) 245{ 246 tcp_rsk(req)->is_mptcp = 1; 247 248 tcp_request_sock_ipv6_ops.init_req(req, sk_listener, skb); 249 250 subflow_init_req(req, sk_listener, skb); 251} 252#endif 253 254/* validate received truncated hmac and create hmac for third ACK */ 255static bool subflow_thmac_valid(struct mptcp_subflow_context *subflow) 256{ 257 u8 hmac[SHA256_DIGEST_SIZE]; 258 u64 thmac; 259 260 subflow_generate_hmac(subflow->remote_key, subflow->local_key, 261 subflow->remote_nonce, subflow->local_nonce, 262 hmac); 263 264 thmac = get_unaligned_be64(hmac); 265 pr_debug("subflow=%p, token=%u, thmac=%llu, subflow->thmac=%llu\n", 266 subflow, subflow->token, 267 (unsigned long long)thmac, 268 (unsigned long long)subflow->thmac); 269 270 return thmac == subflow->thmac; 271} 272 273void mptcp_subflow_reset(struct sock *ssk) 274{ 275 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 276 struct sock *sk = subflow->conn; 277 278 tcp_send_active_reset(ssk, GFP_ATOMIC); 279 tcp_done(ssk); 280 if (!test_and_set_bit(MPTCP_WORK_CLOSE_SUBFLOW, &mptcp_sk(sk)->flags) && 281 schedule_work(&mptcp_sk(sk)->work)) 282 sock_hold(sk); 283} 284 285static void subflow_finish_connect(struct sock *sk, const struct sk_buff *skb) 286{ 287 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 288 struct mptcp_options_received mp_opt; 289 struct sock *parent = subflow->conn; 290 291 subflow->icsk_af_ops->sk_rx_dst_set(sk, skb); 292 293 if (inet_sk_state_load(parent) == TCP_SYN_SENT) { 294 inet_sk_state_store(parent, TCP_ESTABLISHED); 295 parent->sk_state_change(parent); 296 } 297 298 /* be sure no special action on any packet other than syn-ack */ 299 if (subflow->conn_finished) 300 return; 301 302 subflow->rel_write_seq = 1; 303 subflow->conn_finished = 1; 304 subflow->ssn_offset = TCP_SKB_CB(skb)->seq; 305 pr_debug("subflow=%p synack seq=%x", subflow, subflow->ssn_offset); 306 307 mptcp_get_options(skb, &mp_opt); 308 if (subflow->request_mptcp) { 309 if (!mp_opt.mp_capable) { 310 MPTCP_INC_STATS(sock_net(sk), 311 MPTCP_MIB_MPCAPABLEACTIVEFALLBACK); 312 mptcp_do_fallback(sk); 313 pr_fallback(mptcp_sk(subflow->conn)); 314 goto fallback; 315 } 316 317 subflow->mp_capable = 1; 318 subflow->can_ack = 1; 319 subflow->remote_key = mp_opt.sndr_key; 320 pr_debug("subflow=%p, remote_key=%llu", subflow, 321 subflow->remote_key); 322 mptcp_finish_connect(sk); 323 } else if (subflow->request_join) { 324 u8 hmac[SHA256_DIGEST_SIZE]; 325 326 if (!mp_opt.mp_join) 327 goto do_reset; 328 329 subflow->thmac = mp_opt.thmac; 330 subflow->remote_nonce = mp_opt.nonce; 331 pr_debug("subflow=%p, thmac=%llu, remote_nonce=%u", subflow, 332 subflow->thmac, subflow->remote_nonce); 333 334 if (!subflow_thmac_valid(subflow)) { 335 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINACKMAC); 336 goto do_reset; 337 } 338 339 if (!mptcp_finish_join(sk)) 340 goto do_reset; 341 342 subflow_generate_hmac(subflow->local_key, subflow->remote_key, 343 subflow->local_nonce, 344 subflow->remote_nonce, 345 hmac); 346 memcpy(subflow->hmac, hmac, MPTCPOPT_HMAC_LEN); 347 348 subflow->mp_join = 1; 349 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINSYNACKRX); 350 } else if (mptcp_check_fallback(sk)) { 351fallback: 352 mptcp_rcv_space_init(mptcp_sk(parent), sk); 353 } 354 return; 355 356do_reset: 357 mptcp_subflow_reset(sk); 358} 359 360static struct request_sock_ops mptcp_subflow_v4_request_sock_ops __ro_after_init; 361static struct tcp_request_sock_ops subflow_request_sock_ipv4_ops __ro_after_init; 362 363static int subflow_v4_conn_request(struct sock *sk, struct sk_buff *skb) 364{ 365 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 366 367 pr_debug("subflow=%p", subflow); 368 369 /* Never answer to SYNs sent to broadcast or multicast */ 370 if (skb_rtable(skb)->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST)) 371 goto drop; 372 373 return tcp_conn_request(&mptcp_subflow_v4_request_sock_ops, 374 &subflow_request_sock_ipv4_ops, 375 sk, skb); 376drop: 377 tcp_listendrop(sk); 378 return 0; 379} 380 381static void subflow_v4_req_destructor(struct request_sock *req) 382{ 383 subflow_req_destructor(req); 384 tcp_request_sock_ops.destructor(req); 385} 386 387#if IS_ENABLED(CONFIG_MPTCP_IPV6) 388static struct request_sock_ops mptcp_subflow_v6_request_sock_ops __ro_after_init; 389static struct tcp_request_sock_ops subflow_request_sock_ipv6_ops __ro_after_init; 390static struct inet_connection_sock_af_ops subflow_v6_specific __ro_after_init; 391static struct inet_connection_sock_af_ops subflow_v6m_specific __ro_after_init; 392 393static int subflow_v6_conn_request(struct sock *sk, struct sk_buff *skb) 394{ 395 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 396 397 pr_debug("subflow=%p", subflow); 398 399 if (skb->protocol == htons(ETH_P_IP)) 400 return subflow_v4_conn_request(sk, skb); 401 402 if (!ipv6_unicast_destination(skb)) 403 goto drop; 404 405 if (ipv6_addr_v4mapped(&ipv6_hdr(skb)->saddr)) { 406 __IP6_INC_STATS(sock_net(sk), NULL, IPSTATS_MIB_INHDRERRORS); 407 return 0; 408 } 409 410 return tcp_conn_request(&mptcp_subflow_v6_request_sock_ops, 411 &subflow_request_sock_ipv6_ops, sk, skb); 412 413drop: 414 tcp_listendrop(sk); 415 return 0; /* don't send reset */ 416} 417 418static void subflow_v6_req_destructor(struct request_sock *req) 419{ 420 subflow_req_destructor(req); 421 tcp6_request_sock_ops.destructor(req); 422} 423#endif 424 425struct request_sock *mptcp_subflow_reqsk_alloc(const struct request_sock_ops *ops, 426 struct sock *sk_listener, 427 bool attach_listener) 428{ 429 if (ops->family == AF_INET) 430 ops = &mptcp_subflow_v4_request_sock_ops; 431#if IS_ENABLED(CONFIG_MPTCP_IPV6) 432 else if (ops->family == AF_INET6) 433 ops = &mptcp_subflow_v6_request_sock_ops; 434#endif 435 436 return inet_reqsk_alloc(ops, sk_listener, attach_listener); 437} 438EXPORT_SYMBOL(mptcp_subflow_reqsk_alloc); 439 440/* validate hmac received in third ACK */ 441static bool subflow_hmac_valid(const struct request_sock *req, 442 const struct mptcp_options_received *mp_opt) 443{ 444 const struct mptcp_subflow_request_sock *subflow_req; 445 u8 hmac[SHA256_DIGEST_SIZE]; 446 struct mptcp_sock *msk; 447 448 subflow_req = mptcp_subflow_rsk(req); 449 msk = subflow_req->msk; 450 if (!msk) 451 return false; 452 453 subflow_generate_hmac(msk->remote_key, msk->local_key, 454 subflow_req->remote_nonce, 455 subflow_req->local_nonce, hmac); 456 457 return !crypto_memneq(hmac, mp_opt->hmac, MPTCPOPT_HMAC_LEN); 458} 459 460static void mptcp_sock_destruct(struct sock *sk) 461{ 462 /* if new mptcp socket isn't accepted, it is free'd 463 * from the tcp listener sockets request queue, linked 464 * from req->sk. The tcp socket is released. 465 * This calls the ULP release function which will 466 * also remove the mptcp socket, via 467 * sock_put(ctx->conn). 468 * 469 * Problem is that the mptcp socket will be in 470 * ESTABLISHED state and will not have the SOCK_DEAD flag. 471 * Both result in warnings from inet_sock_destruct. 472 */ 473 if ((1 << sk->sk_state) & (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) { 474 sk->sk_state = TCP_CLOSE; 475 WARN_ON_ONCE(sk->sk_socket); 476 sock_orphan(sk); 477 } 478 479 mptcp_destroy_common(mptcp_sk(sk)); 480 inet_sock_destruct(sk); 481} 482 483static void mptcp_force_close(struct sock *sk) 484{ 485 inet_sk_state_store(sk, TCP_CLOSE); 486 sk_common_release(sk); 487} 488 489static void subflow_ulp_fallback(struct sock *sk, 490 struct mptcp_subflow_context *old_ctx) 491{ 492 struct inet_connection_sock *icsk = inet_csk(sk); 493 494 mptcp_subflow_tcp_fallback(sk, old_ctx); 495 icsk->icsk_ulp_ops = NULL; 496 rcu_assign_pointer(icsk->icsk_ulp_data, NULL); 497 tcp_sk(sk)->is_mptcp = 0; 498} 499 500static void subflow_drop_ctx(struct sock *ssk) 501{ 502 struct mptcp_subflow_context *ctx = mptcp_subflow_ctx(ssk); 503 504 if (!ctx) 505 return; 506 507 subflow_ulp_fallback(ssk, ctx); 508 if (ctx->conn) 509 sock_put(ctx->conn); 510 511 kfree_rcu(ctx, rcu); 512} 513 514void mptcp_subflow_fully_established(struct mptcp_subflow_context *subflow, 515 struct mptcp_options_received *mp_opt) 516{ 517 struct mptcp_sock *msk = mptcp_sk(subflow->conn); 518 519 subflow->remote_key = mp_opt->sndr_key; 520 subflow->fully_established = 1; 521 subflow->can_ack = 1; 522 WRITE_ONCE(msk->fully_established, true); 523} 524 525static struct sock *subflow_syn_recv_sock(const struct sock *sk, 526 struct sk_buff *skb, 527 struct request_sock *req, 528 struct dst_entry *dst, 529 struct request_sock *req_unhash, 530 bool *own_req) 531{ 532 struct mptcp_subflow_context *listener = mptcp_subflow_ctx(sk); 533 struct mptcp_subflow_request_sock *subflow_req; 534 struct mptcp_options_received mp_opt; 535 bool fallback, fallback_is_fatal; 536 struct sock *new_msk = NULL; 537 struct sock *child; 538 539 pr_debug("listener=%p, req=%p, conn=%p", listener, req, listener->conn); 540 541 /* After child creation we must look for 'mp_capable' even when options 542 * are not parsed 543 */ 544 mp_opt.mp_capable = 0; 545 546 /* hopefully temporary handling for MP_JOIN+syncookie */ 547 subflow_req = mptcp_subflow_rsk(req); 548 fallback_is_fatal = tcp_rsk(req)->is_mptcp && subflow_req->mp_join; 549 fallback = !tcp_rsk(req)->is_mptcp; 550 if (fallback) 551 goto create_child; 552 553 /* if the sk is MP_CAPABLE, we try to fetch the client key */ 554 if (subflow_req->mp_capable) { 555 /* we can receive and accept an in-window, out-of-order pkt, 556 * which may not carry the MP_CAPABLE opt even on mptcp enabled 557 * paths: always try to extract the peer key, and fallback 558 * for packets missing it. 559 * Even OoO DSS packets coming legitly after dropped or 560 * reordered MPC will cause fallback, but we don't have other 561 * options. 562 */ 563 mptcp_get_options(skb, &mp_opt); 564 if (!mp_opt.mp_capable) { 565 fallback = true; 566 goto create_child; 567 } 568 569 new_msk = mptcp_sk_clone(listener->conn, &mp_opt, req); 570 if (!new_msk) 571 fallback = true; 572 } else if (subflow_req->mp_join) { 573 mptcp_get_options(skb, &mp_opt); 574 if (!mp_opt.mp_join || !subflow_hmac_valid(req, &mp_opt) || 575 !mptcp_can_accept_new_subflow(subflow_req->msk)) { 576 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINACKMAC); 577 fallback = true; 578 } 579 } 580 581create_child: 582 child = listener->icsk_af_ops->syn_recv_sock(sk, skb, req, dst, 583 req_unhash, own_req); 584 585 if (child && *own_req) { 586 struct mptcp_subflow_context *ctx = mptcp_subflow_ctx(child); 587 588 tcp_rsk(req)->drop_req = false; 589 590 /* we need to fallback on ctx allocation failure and on pre-reqs 591 * checking above. In the latter scenario we additionally need 592 * to reset the context to non MPTCP status. 593 */ 594 if (!ctx || fallback) { 595 if (fallback_is_fatal) 596 goto dispose_child; 597 598 subflow_drop_ctx(child); 599 goto out; 600 } 601 602 if (ctx->mp_capable) { 603 /* this can't race with mptcp_close(), as the msk is 604 * not yet exposted to user-space 605 */ 606 inet_sk_state_store((void *)new_msk, TCP_ESTABLISHED); 607 608 /* new mpc subflow takes ownership of the newly 609 * created mptcp socket 610 */ 611 new_msk->sk_destruct = mptcp_sock_destruct; 612 mptcp_pm_new_connection(mptcp_sk(new_msk), 1); 613 mptcp_token_accept(subflow_req, mptcp_sk(new_msk)); 614 ctx->conn = new_msk; 615 new_msk = NULL; 616 617 /* with OoO packets we can reach here without ingress 618 * mpc option 619 */ 620 if (mp_opt.mp_capable) 621 mptcp_subflow_fully_established(ctx, &mp_opt); 622 } else if (ctx->mp_join) { 623 struct mptcp_sock *owner; 624 625 owner = subflow_req->msk; 626 if (!owner) 627 goto dispose_child; 628 629 /* move the msk reference ownership to the subflow */ 630 subflow_req->msk = NULL; 631 ctx->conn = (struct sock *)owner; 632 if (!mptcp_finish_join(child)) 633 goto dispose_child; 634 635 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINACKRX); 636 tcp_rsk(req)->drop_req = true; 637 } 638 } 639 640out: 641 /* dispose of the left over mptcp master, if any */ 642 if (unlikely(new_msk)) 643 mptcp_force_close(new_msk); 644 645 /* check for expected invariant - should never trigger, just help 646 * catching eariler subtle bugs 647 */ 648 WARN_ON_ONCE(child && *own_req && tcp_sk(child)->is_mptcp && 649 (!mptcp_subflow_ctx(child) || 650 !mptcp_subflow_ctx(child)->conn)); 651 return child; 652 653dispose_child: 654 subflow_drop_ctx(child); 655 tcp_rsk(req)->drop_req = true; 656 inet_csk_prepare_for_destroy_sock(child); 657 tcp_done(child); 658 req->rsk_ops->send_reset(sk, skb); 659 660 /* The last child reference will be released by the caller */ 661 return child; 662} 663 664static struct inet_connection_sock_af_ops subflow_specific __ro_after_init; 665 666enum mapping_status { 667 MAPPING_OK, 668 MAPPING_INVALID, 669 MAPPING_EMPTY, 670 MAPPING_DATA_FIN, 671 MAPPING_DUMMY 672}; 673 674static u64 expand_seq(u64 old_seq, u16 old_data_len, u64 seq) 675{ 676 if ((u32)seq == (u32)old_seq) 677 return old_seq; 678 679 /* Assume map covers data not mapped yet. */ 680 return seq | ((old_seq + old_data_len + 1) & GENMASK_ULL(63, 32)); 681} 682 683static void dbg_bad_map(struct mptcp_subflow_context *subflow, u32 ssn) 684{ 685 pr_debug("Bad mapping: ssn=%d map_seq=%d map_data_len=%d", 686 ssn, subflow->map_subflow_seq, subflow->map_data_len); 687} 688 689static bool skb_is_fully_mapped(struct sock *ssk, struct sk_buff *skb) 690{ 691 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 692 unsigned int skb_consumed; 693 694 skb_consumed = tcp_sk(ssk)->copied_seq - TCP_SKB_CB(skb)->seq; 695 if (WARN_ON_ONCE(skb_consumed >= skb->len)) 696 return true; 697 698 return skb->len - skb_consumed <= subflow->map_data_len - 699 mptcp_subflow_get_map_offset(subflow); 700} 701 702static bool validate_mapping(struct sock *ssk, struct sk_buff *skb) 703{ 704 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 705 u32 ssn = tcp_sk(ssk)->copied_seq - subflow->ssn_offset; 706 707 if (unlikely(before(ssn, subflow->map_subflow_seq))) { 708 /* Mapping covers data later in the subflow stream, 709 * currently unsupported. 710 */ 711 dbg_bad_map(subflow, ssn); 712 return false; 713 } 714 if (unlikely(!before(ssn, subflow->map_subflow_seq + 715 subflow->map_data_len))) { 716 /* Mapping does covers past subflow data, invalid */ 717 dbg_bad_map(subflow, ssn); 718 return false; 719 } 720 return true; 721} 722 723static enum mapping_status get_mapping_status(struct sock *ssk, 724 struct mptcp_sock *msk) 725{ 726 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 727 struct mptcp_ext *mpext; 728 struct sk_buff *skb; 729 u16 data_len; 730 u64 map_seq; 731 732 skb = skb_peek(&ssk->sk_receive_queue); 733 if (!skb) 734 return MAPPING_EMPTY; 735 736 if (mptcp_check_fallback(ssk)) 737 return MAPPING_DUMMY; 738 739 mpext = mptcp_get_ext(skb); 740 if (!mpext || !mpext->use_map) { 741 if (!subflow->map_valid && !skb->len) { 742 /* the TCP stack deliver 0 len FIN pkt to the receive 743 * queue, that is the only 0len pkts ever expected here, 744 * and we can admit no mapping only for 0 len pkts 745 */ 746 if (!(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)) 747 WARN_ONCE(1, "0len seq %d:%d flags %x", 748 TCP_SKB_CB(skb)->seq, 749 TCP_SKB_CB(skb)->end_seq, 750 TCP_SKB_CB(skb)->tcp_flags); 751 sk_eat_skb(ssk, skb); 752 return MAPPING_EMPTY; 753 } 754 755 if (!subflow->map_valid) 756 return MAPPING_INVALID; 757 758 goto validate_seq; 759 } 760 761 pr_debug("seq=%llu is64=%d ssn=%u data_len=%u data_fin=%d", 762 mpext->data_seq, mpext->dsn64, mpext->subflow_seq, 763 mpext->data_len, mpext->data_fin); 764 765 data_len = mpext->data_len; 766 if (data_len == 0) { 767 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_INFINITEMAPRX); 768 return MAPPING_INVALID; 769 } 770 771 if (mpext->data_fin == 1) { 772 if (data_len == 1) { 773 bool updated = mptcp_update_rcv_data_fin(msk, mpext->data_seq, 774 mpext->dsn64); 775 pr_debug("DATA_FIN with no payload seq=%llu", mpext->data_seq); 776 if (subflow->map_valid) { 777 /* A DATA_FIN might arrive in a DSS 778 * option before the previous mapping 779 * has been fully consumed. Continue 780 * handling the existing mapping. 781 */ 782 skb_ext_del(skb, SKB_EXT_MPTCP); 783 return MAPPING_OK; 784 } else { 785 if (updated && schedule_work(&msk->work)) 786 sock_hold((struct sock *)msk); 787 788 return MAPPING_DATA_FIN; 789 } 790 } else { 791 u64 data_fin_seq = mpext->data_seq + data_len - 1; 792 793 /* If mpext->data_seq is a 32-bit value, data_fin_seq 794 * must also be limited to 32 bits. 795 */ 796 if (!mpext->dsn64) 797 data_fin_seq &= GENMASK_ULL(31, 0); 798 799 mptcp_update_rcv_data_fin(msk, data_fin_seq, mpext->dsn64); 800 pr_debug("DATA_FIN with mapping seq=%llu dsn64=%d", 801 data_fin_seq, mpext->dsn64); 802 } 803 804 /* Adjust for DATA_FIN using 1 byte of sequence space */ 805 data_len--; 806 } 807 808 if (!mpext->dsn64) { 809 map_seq = expand_seq(subflow->map_seq, subflow->map_data_len, 810 mpext->data_seq); 811 pr_debug("expanded seq=%llu", subflow->map_seq); 812 } else { 813 map_seq = mpext->data_seq; 814 } 815 WRITE_ONCE(mptcp_sk(subflow->conn)->use_64bit_ack, !!mpext->dsn64); 816 817 if (subflow->map_valid) { 818 /* Allow replacing only with an identical map */ 819 if (subflow->map_seq == map_seq && 820 subflow->map_subflow_seq == mpext->subflow_seq && 821 subflow->map_data_len == data_len) { 822 skb_ext_del(skb, SKB_EXT_MPTCP); 823 return MAPPING_OK; 824 } 825 826 /* If this skb data are fully covered by the current mapping, 827 * the new map would need caching, which is not supported 828 */ 829 if (skb_is_fully_mapped(ssk, skb)) { 830 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_DSSNOMATCH); 831 return MAPPING_INVALID; 832 } 833 834 /* will validate the next map after consuming the current one */ 835 return MAPPING_OK; 836 } 837 838 subflow->map_seq = map_seq; 839 subflow->map_subflow_seq = mpext->subflow_seq; 840 subflow->map_data_len = data_len; 841 subflow->map_valid = 1; 842 subflow->mpc_map = mpext->mpc_map; 843 pr_debug("new map seq=%llu subflow_seq=%u data_len=%u", 844 subflow->map_seq, subflow->map_subflow_seq, 845 subflow->map_data_len); 846 847validate_seq: 848 /* we revalidate valid mapping on new skb, because we must ensure 849 * the current skb is completely covered by the available mapping 850 */ 851 if (!validate_mapping(ssk, skb)) 852 return MAPPING_INVALID; 853 854 skb_ext_del(skb, SKB_EXT_MPTCP); 855 return MAPPING_OK; 856} 857 858static void mptcp_subflow_discard_data(struct sock *ssk, struct sk_buff *skb, 859 u64 limit) 860{ 861 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 862 bool fin = TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN; 863 u32 incr; 864 865 incr = limit >= skb->len ? skb->len + fin : limit; 866 867 pr_debug("discarding=%d len=%d seq=%d", incr, skb->len, 868 subflow->map_subflow_seq); 869 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_DUPDATA); 870 tcp_sk(ssk)->copied_seq += incr; 871 if (!before(tcp_sk(ssk)->copied_seq, TCP_SKB_CB(skb)->end_seq)) 872 sk_eat_skb(ssk, skb); 873 if (mptcp_subflow_get_map_offset(subflow) >= subflow->map_data_len) 874 subflow->map_valid = 0; 875 if (incr) 876 tcp_cleanup_rbuf(ssk, incr); 877} 878 879static bool subflow_check_data_avail(struct sock *ssk) 880{ 881 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 882 enum mapping_status status; 883 struct mptcp_sock *msk; 884 struct sk_buff *skb; 885 886 pr_debug("msk=%p ssk=%p data_avail=%d skb=%p", subflow->conn, ssk, 887 subflow->data_avail, skb_peek(&ssk->sk_receive_queue)); 888 if (!skb_peek(&ssk->sk_receive_queue)) 889 subflow->data_avail = 0; 890 if (subflow->data_avail) 891 return true; 892 893 msk = mptcp_sk(subflow->conn); 894 for (;;) { 895 u64 ack_seq; 896 u64 old_ack; 897 898 status = get_mapping_status(ssk, msk); 899 pr_debug("msk=%p ssk=%p status=%d", msk, ssk, status); 900 if (status == MAPPING_INVALID) { 901 ssk->sk_err = EBADMSG; 902 goto fatal; 903 } 904 if (status == MAPPING_DUMMY) { 905 __mptcp_do_fallback(msk); 906 skb = skb_peek(&ssk->sk_receive_queue); 907 subflow->map_valid = 1; 908 subflow->map_seq = READ_ONCE(msk->ack_seq); 909 subflow->map_data_len = skb->len; 910 subflow->map_subflow_seq = tcp_sk(ssk)->copied_seq - 911 subflow->ssn_offset; 912 subflow->data_avail = MPTCP_SUBFLOW_DATA_AVAIL; 913 return true; 914 } 915 916 if (status != MAPPING_OK) 917 return false; 918 919 skb = skb_peek(&ssk->sk_receive_queue); 920 if (WARN_ON_ONCE(!skb)) 921 return false; 922 923 /* if msk lacks the remote key, this subflow must provide an 924 * MP_CAPABLE-based mapping 925 */ 926 if (unlikely(!READ_ONCE(msk->can_ack))) { 927 if (!subflow->mpc_map) { 928 ssk->sk_err = EBADMSG; 929 goto fatal; 930 } 931 WRITE_ONCE(msk->remote_key, subflow->remote_key); 932 WRITE_ONCE(msk->ack_seq, subflow->map_seq); 933 WRITE_ONCE(msk->can_ack, true); 934 } 935 936 old_ack = READ_ONCE(msk->ack_seq); 937 ack_seq = mptcp_subflow_get_mapped_dsn(subflow); 938 pr_debug("msk ack_seq=%llx subflow ack_seq=%llx", old_ack, 939 ack_seq); 940 if (ack_seq == old_ack) { 941 subflow->data_avail = MPTCP_SUBFLOW_DATA_AVAIL; 942 break; 943 } else if (after64(ack_seq, old_ack)) { 944 subflow->data_avail = MPTCP_SUBFLOW_OOO_DATA; 945 break; 946 } 947 948 /* only accept in-sequence mapping. Old values are spurious 949 * retransmission 950 */ 951 mptcp_subflow_discard_data(ssk, skb, old_ack - ack_seq); 952 } 953 return true; 954 955fatal: 956 /* fatal protocol error, close the socket */ 957 /* This barrier is coupled with smp_rmb() in tcp_poll() */ 958 smp_wmb(); 959 ssk->sk_error_report(ssk); 960 tcp_set_state(ssk, TCP_CLOSE); 961 tcp_send_active_reset(ssk, GFP_ATOMIC); 962 subflow->data_avail = 0; 963 return false; 964} 965 966bool mptcp_subflow_data_available(struct sock *sk) 967{ 968 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 969 970 /* check if current mapping is still valid */ 971 if (subflow->map_valid && 972 mptcp_subflow_get_map_offset(subflow) >= subflow->map_data_len) { 973 subflow->map_valid = 0; 974 subflow->data_avail = 0; 975 976 pr_debug("Done with mapping: seq=%u data_len=%u", 977 subflow->map_subflow_seq, 978 subflow->map_data_len); 979 } 980 981 return subflow_check_data_avail(sk); 982} 983 984/* If ssk has an mptcp parent socket, use the mptcp rcvbuf occupancy, 985 * not the ssk one. 986 * 987 * In mptcp, rwin is about the mptcp-level connection data. 988 * 989 * Data that is still on the ssk rx queue can thus be ignored, 990 * as far as mptcp peer is concerened that data is still inflight. 991 * DSS ACK is updated when skb is moved to the mptcp rx queue. 992 */ 993void mptcp_space(const struct sock *ssk, int *space, int *full_space) 994{ 995 const struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 996 const struct sock *sk = subflow->conn; 997 998 *space = tcp_space(sk); 999 *full_space = tcp_full_space(sk); 1000} 1001 1002static void subflow_data_ready(struct sock *sk) 1003{ 1004 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 1005 u16 state = 1 << inet_sk_state_load(sk); 1006 struct sock *parent = subflow->conn; 1007 struct mptcp_sock *msk; 1008 1009 msk = mptcp_sk(parent); 1010 if (state & TCPF_LISTEN) { 1011 /* MPJ subflow are removed from accept queue before reaching here, 1012 * avoid stray wakeups 1013 */ 1014 if (reqsk_queue_empty(&inet_csk(sk)->icsk_accept_queue)) 1015 return; 1016 1017 set_bit(MPTCP_DATA_READY, &msk->flags); 1018 parent->sk_data_ready(parent); 1019 return; 1020 } 1021 1022 WARN_ON_ONCE(!__mptcp_check_fallback(msk) && !subflow->mp_capable && 1023 !subflow->mp_join && !(state & TCPF_CLOSE)); 1024 1025 if (mptcp_subflow_data_available(sk)) 1026 mptcp_data_ready(parent, sk); 1027} 1028 1029static void subflow_write_space(struct sock *sk) 1030{ 1031 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 1032 struct sock *parent = subflow->conn; 1033 1034 if (!sk_stream_is_writeable(sk)) 1035 return; 1036 1037 if (sk_stream_is_writeable(parent)) { 1038 set_bit(MPTCP_SEND_SPACE, &mptcp_sk(parent)->flags); 1039 smp_mb__after_atomic(); 1040 /* set SEND_SPACE before sk_stream_write_space clears NOSPACE */ 1041 sk_stream_write_space(parent); 1042 } 1043} 1044 1045static const struct inet_connection_sock_af_ops * 1046subflow_default_af_ops(struct sock *sk) 1047{ 1048#if IS_ENABLED(CONFIG_MPTCP_IPV6) 1049 if (sk->sk_family == AF_INET6) 1050 return &subflow_v6_specific; 1051#endif 1052 return &subflow_specific; 1053} 1054 1055#if IS_ENABLED(CONFIG_MPTCP_IPV6) 1056void mptcpv6_handle_mapped(struct sock *sk, bool mapped) 1057{ 1058 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 1059 struct inet_connection_sock *icsk = inet_csk(sk); 1060 const struct inet_connection_sock_af_ops *target; 1061 1062 target = mapped ? &subflow_v6m_specific : subflow_default_af_ops(sk); 1063 1064 pr_debug("subflow=%p family=%d ops=%p target=%p mapped=%d", 1065 subflow, sk->sk_family, icsk->icsk_af_ops, target, mapped); 1066 1067 if (likely(icsk->icsk_af_ops == target)) 1068 return; 1069 1070 subflow->icsk_af_ops = icsk->icsk_af_ops; 1071 icsk->icsk_af_ops = target; 1072} 1073#endif 1074 1075static void mptcp_info2sockaddr(const struct mptcp_addr_info *info, 1076 struct sockaddr_storage *addr) 1077{ 1078 memset(addr, 0, sizeof(*addr)); 1079 addr->ss_family = info->family; 1080 if (addr->ss_family == AF_INET) { 1081 struct sockaddr_in *in_addr = (struct sockaddr_in *)addr; 1082 1083 in_addr->sin_addr = info->addr; 1084 in_addr->sin_port = info->port; 1085 } 1086#if IS_ENABLED(CONFIG_MPTCP_IPV6) 1087 else if (addr->ss_family == AF_INET6) { 1088 struct sockaddr_in6 *in6_addr = (struct sockaddr_in6 *)addr; 1089 1090 in6_addr->sin6_addr = info->addr6; 1091 in6_addr->sin6_port = info->port; 1092 } 1093#endif 1094} 1095 1096int __mptcp_subflow_connect(struct sock *sk, const struct mptcp_addr_info *loc, 1097 const struct mptcp_addr_info *remote) 1098{ 1099 struct mptcp_sock *msk = mptcp_sk(sk); 1100 struct mptcp_subflow_context *subflow; 1101 struct sockaddr_storage addr; 1102 int remote_id = remote->id; 1103 int local_id = loc->id; 1104 struct socket *sf; 1105 struct sock *ssk; 1106 u32 remote_token; 1107 int addrlen; 1108 int err; 1109 1110 if (!mptcp_is_fully_established(sk)) 1111 return -ENOTCONN; 1112 1113 err = mptcp_subflow_create_socket(sk, &sf); 1114 if (err) 1115 return err; 1116 1117 ssk = sf->sk; 1118 subflow = mptcp_subflow_ctx(ssk); 1119 do { 1120 get_random_bytes(&subflow->local_nonce, sizeof(u32)); 1121 } while (!subflow->local_nonce); 1122 1123 if (!local_id) { 1124 err = mptcp_pm_get_local_id(msk, (struct sock_common *)ssk); 1125 if (err < 0) 1126 goto failed; 1127 1128 local_id = err; 1129 } 1130 1131 subflow->remote_key = msk->remote_key; 1132 subflow->local_key = msk->local_key; 1133 subflow->token = msk->token; 1134 mptcp_info2sockaddr(loc, &addr); 1135 1136 addrlen = sizeof(struct sockaddr_in); 1137#if IS_ENABLED(CONFIG_MPTCP_IPV6) 1138 if (loc->family == AF_INET6) 1139 addrlen = sizeof(struct sockaddr_in6); 1140#endif 1141 ssk->sk_bound_dev_if = loc->ifindex; 1142 err = kernel_bind(sf, (struct sockaddr *)&addr, addrlen); 1143 if (err) 1144 goto failed; 1145 1146 mptcp_crypto_key_sha(subflow->remote_key, &remote_token, NULL); 1147 pr_debug("msk=%p remote_token=%u local_id=%d remote_id=%d", msk, 1148 remote_token, local_id, remote_id); 1149 subflow->remote_token = remote_token; 1150 subflow->local_id = local_id; 1151 subflow->remote_id = remote_id; 1152 subflow->request_join = 1; 1153 subflow->request_bkup = !!(loc->flags & MPTCP_PM_ADDR_FLAG_BACKUP); 1154 mptcp_info2sockaddr(remote, &addr); 1155 1156 err = kernel_connect(sf, (struct sockaddr *)&addr, addrlen, O_NONBLOCK); 1157 if (err && err != -EINPROGRESS) 1158 goto failed; 1159 1160 spin_lock_bh(&msk->join_list_lock); 1161 list_add_tail(&subflow->node, &msk->join_list); 1162 spin_unlock_bh(&msk->join_list_lock); 1163 1164 return err; 1165 1166failed: 1167 sock_release(sf); 1168 return err; 1169} 1170 1171int mptcp_subflow_create_socket(struct sock *sk, struct socket **new_sock) 1172{ 1173 struct mptcp_subflow_context *subflow; 1174 struct net *net = sock_net(sk); 1175 struct socket *sf; 1176 int err; 1177 1178 /* un-accepted server sockets can reach here - on bad configuration 1179 * bail early to avoid greater trouble later 1180 */ 1181 if (unlikely(!sk->sk_socket)) 1182 return -EINVAL; 1183 1184 err = sock_create_kern(net, sk->sk_family, SOCK_STREAM, IPPROTO_TCP, 1185 &sf); 1186 if (err) 1187 return err; 1188 1189 lock_sock(sf->sk); 1190 1191 /* kernel sockets do not by default acquire net ref, but TCP timer 1192 * needs it. 1193 */ 1194 sf->sk->sk_net_refcnt = 1; 1195 get_net(net); 1196#ifdef CONFIG_PROC_FS 1197 this_cpu_add(*net->core.sock_inuse, 1); 1198#endif 1199 err = tcp_set_ulp(sf->sk, "mptcp"); 1200 release_sock(sf->sk); 1201 1202 if (err) { 1203 sock_release(sf); 1204 return err; 1205 } 1206 1207 /* the newly created socket really belongs to the owning MPTCP master 1208 * socket, even if for additional subflows the allocation is performed 1209 * by a kernel workqueue. Adjust inode references, so that the 1210 * procfs/diag interaces really show this one belonging to the correct 1211 * user. 1212 */ 1213 SOCK_INODE(sf)->i_ino = SOCK_INODE(sk->sk_socket)->i_ino; 1214 SOCK_INODE(sf)->i_uid = SOCK_INODE(sk->sk_socket)->i_uid; 1215 SOCK_INODE(sf)->i_gid = SOCK_INODE(sk->sk_socket)->i_gid; 1216 1217 subflow = mptcp_subflow_ctx(sf->sk); 1218 pr_debug("subflow=%p", subflow); 1219 1220 *new_sock = sf; 1221 sock_hold(sk); 1222 subflow->conn = sk; 1223 1224 return 0; 1225} 1226 1227static struct mptcp_subflow_context *subflow_create_ctx(struct sock *sk, 1228 gfp_t priority) 1229{ 1230 struct inet_connection_sock *icsk = inet_csk(sk); 1231 struct mptcp_subflow_context *ctx; 1232 1233 ctx = kzalloc(sizeof(*ctx), priority); 1234 if (!ctx) 1235 return NULL; 1236 1237 rcu_assign_pointer(icsk->icsk_ulp_data, ctx); 1238 INIT_LIST_HEAD(&ctx->node); 1239 1240 pr_debug("subflow=%p", ctx); 1241 1242 ctx->tcp_sock = sk; 1243 1244 return ctx; 1245} 1246 1247static void __subflow_state_change(struct sock *sk) 1248{ 1249 struct socket_wq *wq; 1250 1251 rcu_read_lock(); 1252 wq = rcu_dereference(sk->sk_wq); 1253 if (skwq_has_sleeper(wq)) 1254 wake_up_interruptible_all(&wq->wait); 1255 rcu_read_unlock(); 1256} 1257 1258static bool subflow_is_done(const struct sock *sk) 1259{ 1260 return sk->sk_shutdown & RCV_SHUTDOWN || sk->sk_state == TCP_CLOSE; 1261} 1262 1263static void subflow_state_change(struct sock *sk) 1264{ 1265 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 1266 struct sock *parent = subflow->conn; 1267 1268 __subflow_state_change(sk); 1269 1270 if (subflow_simultaneous_connect(sk)) { 1271 mptcp_do_fallback(sk); 1272 mptcp_rcv_space_init(mptcp_sk(parent), sk); 1273 pr_fallback(mptcp_sk(parent)); 1274 subflow->conn_finished = 1; 1275 if (inet_sk_state_load(parent) == TCP_SYN_SENT) { 1276 inet_sk_state_store(parent, TCP_ESTABLISHED); 1277 parent->sk_state_change(parent); 1278 } 1279 } 1280 1281 /* as recvmsg() does not acquire the subflow socket for ssk selection 1282 * a fin packet carrying a DSS can be unnoticed if we don't trigger 1283 * the data available machinery here. 1284 */ 1285 if (mptcp_subflow_data_available(sk)) 1286 mptcp_data_ready(parent, sk); 1287 1288 if (__mptcp_check_fallback(mptcp_sk(parent)) && 1289 !(parent->sk_shutdown & RCV_SHUTDOWN) && 1290 !subflow->rx_eof && subflow_is_done(sk)) { 1291 subflow->rx_eof = 1; 1292 mptcp_subflow_eof(parent); 1293 } 1294} 1295 1296static int subflow_ulp_init(struct sock *sk) 1297{ 1298 struct inet_connection_sock *icsk = inet_csk(sk); 1299 struct mptcp_subflow_context *ctx; 1300 struct tcp_sock *tp = tcp_sk(sk); 1301 int err = 0; 1302 1303 /* disallow attaching ULP to a socket unless it has been 1304 * created with sock_create_kern() 1305 */ 1306 if (!sk->sk_kern_sock) { 1307 err = -EOPNOTSUPP; 1308 goto out; 1309 } 1310 1311 ctx = subflow_create_ctx(sk, GFP_KERNEL); 1312 if (!ctx) { 1313 err = -ENOMEM; 1314 goto out; 1315 } 1316 1317 pr_debug("subflow=%p, family=%d", ctx, sk->sk_family); 1318 1319 tp->is_mptcp = 1; 1320 ctx->icsk_af_ops = icsk->icsk_af_ops; 1321 icsk->icsk_af_ops = subflow_default_af_ops(sk); 1322 ctx->tcp_data_ready = sk->sk_data_ready; 1323 ctx->tcp_state_change = sk->sk_state_change; 1324 ctx->tcp_write_space = sk->sk_write_space; 1325 sk->sk_data_ready = subflow_data_ready; 1326 sk->sk_write_space = subflow_write_space; 1327 sk->sk_state_change = subflow_state_change; 1328out: 1329 return err; 1330} 1331 1332static void subflow_ulp_release(struct sock *sk) 1333{ 1334 struct mptcp_subflow_context *ctx = mptcp_subflow_ctx(sk); 1335 1336 if (!ctx) 1337 return; 1338 1339 if (ctx->conn) 1340 sock_put(ctx->conn); 1341 1342 kfree_rcu(ctx, rcu); 1343} 1344 1345static void subflow_ulp_clone(const struct request_sock *req, 1346 struct sock *newsk, 1347 const gfp_t priority) 1348{ 1349 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); 1350 struct mptcp_subflow_context *old_ctx = mptcp_subflow_ctx(newsk); 1351 struct mptcp_subflow_context *new_ctx; 1352 1353 if (!tcp_rsk(req)->is_mptcp || 1354 (!subflow_req->mp_capable && !subflow_req->mp_join)) { 1355 subflow_ulp_fallback(newsk, old_ctx); 1356 return; 1357 } 1358 1359 new_ctx = subflow_create_ctx(newsk, priority); 1360 if (!new_ctx) { 1361 subflow_ulp_fallback(newsk, old_ctx); 1362 return; 1363 } 1364 1365 new_ctx->conn_finished = 1; 1366 new_ctx->icsk_af_ops = old_ctx->icsk_af_ops; 1367 new_ctx->tcp_data_ready = old_ctx->tcp_data_ready; 1368 new_ctx->tcp_state_change = old_ctx->tcp_state_change; 1369 new_ctx->tcp_write_space = old_ctx->tcp_write_space; 1370 new_ctx->rel_write_seq = 1; 1371 new_ctx->tcp_sock = newsk; 1372 1373 if (subflow_req->mp_capable) { 1374 /* see comments in subflow_syn_recv_sock(), MPTCP connection 1375 * is fully established only after we receive the remote key 1376 */ 1377 new_ctx->mp_capable = 1; 1378 new_ctx->local_key = subflow_req->local_key; 1379 new_ctx->token = subflow_req->token; 1380 new_ctx->ssn_offset = subflow_req->ssn_offset; 1381 new_ctx->idsn = subflow_req->idsn; 1382 } else if (subflow_req->mp_join) { 1383 new_ctx->ssn_offset = subflow_req->ssn_offset; 1384 new_ctx->mp_join = 1; 1385 new_ctx->fully_established = 1; 1386 new_ctx->backup = subflow_req->backup; 1387 new_ctx->local_id = subflow_req->local_id; 1388 new_ctx->remote_id = subflow_req->remote_id; 1389 new_ctx->token = subflow_req->token; 1390 new_ctx->thmac = subflow_req->thmac; 1391 } 1392} 1393 1394static struct tcp_ulp_ops subflow_ulp_ops __read_mostly = { 1395 .name = "mptcp", 1396 .owner = THIS_MODULE, 1397 .init = subflow_ulp_init, 1398 .release = subflow_ulp_release, 1399 .clone = subflow_ulp_clone, 1400}; 1401 1402static int subflow_ops_init(struct request_sock_ops *subflow_ops) 1403{ 1404 subflow_ops->obj_size = sizeof(struct mptcp_subflow_request_sock); 1405 1406 subflow_ops->slab = kmem_cache_create(subflow_ops->slab_name, 1407 subflow_ops->obj_size, 0, 1408 SLAB_ACCOUNT | 1409 SLAB_TYPESAFE_BY_RCU, 1410 NULL); 1411 if (!subflow_ops->slab) 1412 return -ENOMEM; 1413 1414 return 0; 1415} 1416 1417void __init mptcp_subflow_init(void) 1418{ 1419 mptcp_subflow_v4_request_sock_ops = tcp_request_sock_ops; 1420 mptcp_subflow_v4_request_sock_ops.slab_name = "request_sock_subflow_v4"; 1421 mptcp_subflow_v4_request_sock_ops.destructor = subflow_v4_req_destructor; 1422 1423 if (subflow_ops_init(&mptcp_subflow_v4_request_sock_ops) != 0) 1424 panic("MPTCP: failed to init subflow v4 request sock ops\n"); 1425 1426 subflow_request_sock_ipv4_ops = tcp_request_sock_ipv4_ops; 1427 subflow_request_sock_ipv4_ops.init_req = subflow_v4_init_req; 1428 1429 subflow_specific = ipv4_specific; 1430 subflow_specific.conn_request = subflow_v4_conn_request; 1431 subflow_specific.syn_recv_sock = subflow_syn_recv_sock; 1432 subflow_specific.sk_rx_dst_set = subflow_finish_connect; 1433 1434#if IS_ENABLED(CONFIG_MPTCP_IPV6) 1435 /* In struct mptcp_subflow_request_sock, we assume the TCP request sock 1436 * structures for v4 and v6 have the same size. It should not changed in 1437 * the future but better to make sure to be warned if it is no longer 1438 * the case. 1439 */ 1440 BUILD_BUG_ON(sizeof(struct tcp_request_sock) != sizeof(struct tcp6_request_sock)); 1441 1442 mptcp_subflow_v6_request_sock_ops = tcp6_request_sock_ops; 1443 mptcp_subflow_v6_request_sock_ops.slab_name = "request_sock_subflow_v6"; 1444 mptcp_subflow_v6_request_sock_ops.destructor = subflow_v6_req_destructor; 1445 1446 if (subflow_ops_init(&mptcp_subflow_v6_request_sock_ops) != 0) 1447 panic("MPTCP: failed to init subflow v6 request sock ops\n"); 1448 1449 subflow_request_sock_ipv6_ops = tcp_request_sock_ipv6_ops; 1450 subflow_request_sock_ipv6_ops.init_req = subflow_v6_init_req; 1451 1452 subflow_v6_specific = ipv6_specific; 1453 subflow_v6_specific.conn_request = subflow_v6_conn_request; 1454 subflow_v6_specific.syn_recv_sock = subflow_syn_recv_sock; 1455 subflow_v6_specific.sk_rx_dst_set = subflow_finish_connect; 1456 1457 subflow_v6m_specific = subflow_v6_specific; 1458 subflow_v6m_specific.queue_xmit = ipv4_specific.queue_xmit; 1459 subflow_v6m_specific.send_check = ipv4_specific.send_check; 1460 subflow_v6m_specific.net_header_len = ipv4_specific.net_header_len; 1461 subflow_v6m_specific.mtu_reduced = ipv4_specific.mtu_reduced; 1462 subflow_v6m_specific.net_frag_header_len = 0; 1463#endif 1464 1465 mptcp_diag_subflow_init(&subflow_ulp_ops); 1466 1467 if (tcp_register_ulp(&subflow_ulp_ops) != 0) 1468 panic("MPTCP: failed to register subflows to ULP\n"); 1469} 1470