1/* 2 * Copyright (c) 2016-2017, Mellanox Technologies. All rights reserved. 3 * Copyright (c) 2016-2017, Dave Watson <davejwatson@fb.com>. All rights reserved. 4 * Copyright (c) 2016-2017, Lance Chao <lancerchao@fb.com>. All rights reserved. 5 * Copyright (c) 2016, Fridolin Pokorny <fridolin.pokorny@gmail.com>. All rights reserved. 6 * Copyright (c) 2016, Nikos Mavrogiannopoulos <nmav@gnutls.org>. All rights reserved. 7 * Copyright (c) 2018, Covalent IO, Inc. http://covalent.io 8 * 9 * This software is available to you under a choice of one of two 10 * licenses. You may choose to be licensed under the terms of the GNU 11 * General Public License (GPL) Version 2, available from the file 12 * COPYING in the main directory of this source tree, or the 13 * OpenIB.org BSD license below: 14 * 15 * Redistribution and use in source and binary forms, with or 16 * without modification, are permitted provided that the following 17 * conditions are met: 18 * 19 * - Redistributions of source code must retain the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer. 22 * 23 * - Redistributions in binary form must reproduce the above 24 * copyright notice, this list of conditions and the following 25 * disclaimer in the documentation and/or other materials 26 * provided with the distribution. 27 * 28 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 29 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 30 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 31 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 32 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 33 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 34 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 35 * SOFTWARE. 36 */ 37 38#include <linux/bug.h> 39#include <linux/sched/signal.h> 40#include <linux/module.h> 41#include <linux/splice.h> 42#include <crypto/aead.h> 43 44#include <net/strparser.h> 45#include <net/tls.h> 46 47noinline void tls_err_abort(struct sock *sk, int err) 48{ 49 WARN_ON_ONCE(err >= 0); 50 /* sk->sk_err should contain a positive error code. */ 51 sk->sk_err = -err; 52 sk->sk_error_report(sk); 53} 54 55static int __skb_nsg(struct sk_buff *skb, int offset, int len, 56 unsigned int recursion_level) 57{ 58 int start = skb_headlen(skb); 59 int i, chunk = start - offset; 60 struct sk_buff *frag_iter; 61 int elt = 0; 62 63 if (unlikely(recursion_level >= 24)) 64 return -EMSGSIZE; 65 66 if (chunk > 0) { 67 if (chunk > len) 68 chunk = len; 69 elt++; 70 len -= chunk; 71 if (len == 0) 72 return elt; 73 offset += chunk; 74 } 75 76 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 77 int end; 78 79 WARN_ON(start > offset + len); 80 81 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]); 82 chunk = end - offset; 83 if (chunk > 0) { 84 if (chunk > len) 85 chunk = len; 86 elt++; 87 len -= chunk; 88 if (len == 0) 89 return elt; 90 offset += chunk; 91 } 92 start = end; 93 } 94 95 if (unlikely(skb_has_frag_list(skb))) { 96 skb_walk_frags(skb, frag_iter) { 97 int end, ret; 98 99 WARN_ON(start > offset + len); 100 101 end = start + frag_iter->len; 102 chunk = end - offset; 103 if (chunk > 0) { 104 if (chunk > len) 105 chunk = len; 106 ret = __skb_nsg(frag_iter, offset - start, chunk, 107 recursion_level + 1); 108 if (unlikely(ret < 0)) 109 return ret; 110 elt += ret; 111 len -= chunk; 112 if (len == 0) 113 return elt; 114 offset += chunk; 115 } 116 start = end; 117 } 118 } 119 BUG_ON(len); 120 return elt; 121} 122 123/* Return the number of scatterlist elements required to completely map the 124 * skb, or -EMSGSIZE if the recursion depth is exceeded. 125 */ 126static int skb_nsg(struct sk_buff *skb, int offset, int len) 127{ 128 return __skb_nsg(skb, offset, len, 0); 129} 130 131static int padding_length(struct tls_sw_context_rx *ctx, 132 struct tls_prot_info *prot, struct sk_buff *skb) 133{ 134 struct strp_msg *rxm = strp_msg(skb); 135 int sub = 0; 136 137 /* Determine zero-padding length */ 138 if (prot->version == TLS_1_3_VERSION) { 139 char content_type = 0; 140 int err; 141 int back = 17; 142 143 while (content_type == 0) { 144 if (back > rxm->full_len - prot->prepend_size) 145 return -EBADMSG; 146 err = skb_copy_bits(skb, 147 rxm->offset + rxm->full_len - back, 148 &content_type, 1); 149 if (err) 150 return err; 151 if (content_type) 152 break; 153 sub++; 154 back++; 155 } 156 ctx->control = content_type; 157 } 158 return sub; 159} 160 161static void tls_decrypt_done(struct crypto_async_request *req, int err) 162{ 163 struct aead_request *aead_req = (struct aead_request *)req; 164 struct scatterlist *sgout = aead_req->dst; 165 struct scatterlist *sgin = aead_req->src; 166 struct tls_sw_context_rx *ctx; 167 struct tls_context *tls_ctx; 168 struct tls_prot_info *prot; 169 struct scatterlist *sg; 170 struct sk_buff *skb; 171 unsigned int pages; 172 int pending; 173 174 skb = (struct sk_buff *)req->data; 175 tls_ctx = tls_get_ctx(skb->sk); 176 ctx = tls_sw_ctx_rx(tls_ctx); 177 prot = &tls_ctx->prot_info; 178 179 /* Propagate if there was an err */ 180 if (err) { 181 if (err == -EBADMSG) 182 TLS_INC_STATS(sock_net(skb->sk), 183 LINUX_MIB_TLSDECRYPTERROR); 184 ctx->async_wait.err = err; 185 tls_err_abort(skb->sk, err); 186 } else { 187 struct strp_msg *rxm = strp_msg(skb); 188 int pad; 189 190 pad = padding_length(ctx, prot, skb); 191 if (pad < 0) { 192 ctx->async_wait.err = pad; 193 tls_err_abort(skb->sk, pad); 194 } else { 195 rxm->full_len -= pad; 196 rxm->offset += prot->prepend_size; 197 rxm->full_len -= prot->overhead_size; 198 } 199 } 200 201 /* After using skb->sk to propagate sk through crypto async callback 202 * we need to NULL it again. 203 */ 204 skb->sk = NULL; 205 206 207 /* Free the destination pages if skb was not decrypted inplace */ 208 if (sgout != sgin) { 209 /* Skip the first S/G entry as it points to AAD */ 210 for_each_sg(sg_next(sgout), sg, UINT_MAX, pages) { 211 if (!sg) 212 break; 213 put_page(sg_page(sg)); 214 } 215 } 216 217 kfree(aead_req); 218 219 spin_lock_bh(&ctx->decrypt_compl_lock); 220 pending = atomic_dec_return(&ctx->decrypt_pending); 221 222 if (!pending && ctx->async_notify) 223 complete(&ctx->async_wait.completion); 224 spin_unlock_bh(&ctx->decrypt_compl_lock); 225} 226 227static int tls_do_decryption(struct sock *sk, 228 struct sk_buff *skb, 229 struct scatterlist *sgin, 230 struct scatterlist *sgout, 231 char *iv_recv, 232 size_t data_len, 233 struct aead_request *aead_req, 234 bool async) 235{ 236 struct tls_context *tls_ctx = tls_get_ctx(sk); 237 struct tls_prot_info *prot = &tls_ctx->prot_info; 238 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 239 int ret; 240 241 aead_request_set_tfm(aead_req, ctx->aead_recv); 242 aead_request_set_ad(aead_req, prot->aad_size); 243 aead_request_set_crypt(aead_req, sgin, sgout, 244 data_len + prot->tag_size, 245 (u8 *)iv_recv); 246 247 if (async) { 248 /* Using skb->sk to push sk through to crypto async callback 249 * handler. This allows propagating errors up to the socket 250 * if needed. It _must_ be cleared in the async handler 251 * before consume_skb is called. We _know_ skb->sk is NULL 252 * because it is a clone from strparser. 253 */ 254 skb->sk = sk; 255 aead_request_set_callback(aead_req, 256 CRYPTO_TFM_REQ_MAY_BACKLOG, 257 tls_decrypt_done, skb); 258 atomic_inc(&ctx->decrypt_pending); 259 } else { 260 aead_request_set_callback(aead_req, 261 CRYPTO_TFM_REQ_MAY_BACKLOG, 262 crypto_req_done, &ctx->async_wait); 263 } 264 265 ret = crypto_aead_decrypt(aead_req); 266 if (ret == -EINPROGRESS) { 267 if (async) 268 return ret; 269 270 ret = crypto_wait_req(ret, &ctx->async_wait); 271 } 272 273 if (async) 274 atomic_dec(&ctx->decrypt_pending); 275 276 return ret; 277} 278 279static void tls_trim_both_msgs(struct sock *sk, int target_size) 280{ 281 struct tls_context *tls_ctx = tls_get_ctx(sk); 282 struct tls_prot_info *prot = &tls_ctx->prot_info; 283 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 284 struct tls_rec *rec = ctx->open_rec; 285 286 sk_msg_trim(sk, &rec->msg_plaintext, target_size); 287 if (target_size > 0) 288 target_size += prot->overhead_size; 289 sk_msg_trim(sk, &rec->msg_encrypted, target_size); 290} 291 292static int tls_alloc_encrypted_msg(struct sock *sk, int len) 293{ 294 struct tls_context *tls_ctx = tls_get_ctx(sk); 295 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 296 struct tls_rec *rec = ctx->open_rec; 297 struct sk_msg *msg_en = &rec->msg_encrypted; 298 299 return sk_msg_alloc(sk, msg_en, len, 0); 300} 301 302static int tls_clone_plaintext_msg(struct sock *sk, int required) 303{ 304 struct tls_context *tls_ctx = tls_get_ctx(sk); 305 struct tls_prot_info *prot = &tls_ctx->prot_info; 306 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 307 struct tls_rec *rec = ctx->open_rec; 308 struct sk_msg *msg_pl = &rec->msg_plaintext; 309 struct sk_msg *msg_en = &rec->msg_encrypted; 310 int skip, len; 311 312 /* We add page references worth len bytes from encrypted sg 313 * at the end of plaintext sg. It is guaranteed that msg_en 314 * has enough required room (ensured by caller). 315 */ 316 len = required - msg_pl->sg.size; 317 318 /* Skip initial bytes in msg_en's data to be able to use 319 * same offset of both plain and encrypted data. 320 */ 321 skip = prot->prepend_size + msg_pl->sg.size; 322 323 return sk_msg_clone(sk, msg_pl, msg_en, skip, len); 324} 325 326static struct tls_rec *tls_get_rec(struct sock *sk) 327{ 328 struct tls_context *tls_ctx = tls_get_ctx(sk); 329 struct tls_prot_info *prot = &tls_ctx->prot_info; 330 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 331 struct sk_msg *msg_pl, *msg_en; 332 struct tls_rec *rec; 333 int mem_size; 334 335 mem_size = sizeof(struct tls_rec) + crypto_aead_reqsize(ctx->aead_send); 336 337 rec = kzalloc(mem_size, sk->sk_allocation); 338 if (!rec) 339 return NULL; 340 341 msg_pl = &rec->msg_plaintext; 342 msg_en = &rec->msg_encrypted; 343 344 sk_msg_init(msg_pl); 345 sk_msg_init(msg_en); 346 347 sg_init_table(rec->sg_aead_in, 2); 348 sg_set_buf(&rec->sg_aead_in[0], rec->aad_space, prot->aad_size); 349 sg_unmark_end(&rec->sg_aead_in[1]); 350 351 sg_init_table(rec->sg_aead_out, 2); 352 sg_set_buf(&rec->sg_aead_out[0], rec->aad_space, prot->aad_size); 353 sg_unmark_end(&rec->sg_aead_out[1]); 354 355 return rec; 356} 357 358static void tls_free_rec(struct sock *sk, struct tls_rec *rec) 359{ 360 sk_msg_free(sk, &rec->msg_encrypted); 361 sk_msg_free(sk, &rec->msg_plaintext); 362 kfree(rec); 363} 364 365static void tls_free_open_rec(struct sock *sk) 366{ 367 struct tls_context *tls_ctx = tls_get_ctx(sk); 368 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 369 struct tls_rec *rec = ctx->open_rec; 370 371 if (rec) { 372 tls_free_rec(sk, rec); 373 ctx->open_rec = NULL; 374 } 375} 376 377int tls_tx_records(struct sock *sk, int flags) 378{ 379 struct tls_context *tls_ctx = tls_get_ctx(sk); 380 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 381 struct tls_rec *rec, *tmp; 382 struct sk_msg *msg_en; 383 int tx_flags, rc = 0; 384 385 if (tls_is_partially_sent_record(tls_ctx)) { 386 rec = list_first_entry(&ctx->tx_list, 387 struct tls_rec, list); 388 389 if (flags == -1) 390 tx_flags = rec->tx_flags; 391 else 392 tx_flags = flags; 393 394 rc = tls_push_partial_record(sk, tls_ctx, tx_flags); 395 if (rc) 396 goto tx_err; 397 398 /* Full record has been transmitted. 399 * Remove the head of tx_list 400 */ 401 list_del(&rec->list); 402 sk_msg_free(sk, &rec->msg_plaintext); 403 kfree(rec); 404 } 405 406 /* Tx all ready records */ 407 list_for_each_entry_safe(rec, tmp, &ctx->tx_list, list) { 408 if (READ_ONCE(rec->tx_ready)) { 409 if (flags == -1) 410 tx_flags = rec->tx_flags; 411 else 412 tx_flags = flags; 413 414 msg_en = &rec->msg_encrypted; 415 rc = tls_push_sg(sk, tls_ctx, 416 &msg_en->sg.data[msg_en->sg.curr], 417 0, tx_flags); 418 if (rc) 419 goto tx_err; 420 421 list_del(&rec->list); 422 sk_msg_free(sk, &rec->msg_plaintext); 423 kfree(rec); 424 } else { 425 break; 426 } 427 } 428 429tx_err: 430 if (rc < 0 && rc != -EAGAIN) 431 tls_err_abort(sk, -EBADMSG); 432 433 return rc; 434} 435 436static void tls_encrypt_done(struct crypto_async_request *req, int err) 437{ 438 struct aead_request *aead_req = (struct aead_request *)req; 439 struct sock *sk = req->data; 440 struct tls_context *tls_ctx = tls_get_ctx(sk); 441 struct tls_prot_info *prot = &tls_ctx->prot_info; 442 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 443 struct scatterlist *sge; 444 struct sk_msg *msg_en; 445 struct tls_rec *rec; 446 int pending; 447 448 rec = container_of(aead_req, struct tls_rec, aead_req); 449 msg_en = &rec->msg_encrypted; 450 451 sge = sk_msg_elem(msg_en, msg_en->sg.curr); 452 sge->offset -= prot->prepend_size; 453 sge->length += prot->prepend_size; 454 455 /* Check if error is previously set on socket */ 456 if (err || sk->sk_err) { 457 rec = NULL; 458 459 /* If err is already set on socket, return the same code */ 460 if (sk->sk_err) { 461 ctx->async_wait.err = -sk->sk_err; 462 } else { 463 ctx->async_wait.err = err; 464 tls_err_abort(sk, err); 465 } 466 } 467 468 if (rec) { 469 struct tls_rec *first_rec; 470 471 /* Mark the record as ready for transmission */ 472 smp_store_mb(rec->tx_ready, true); 473 474 /* If received record is at head of tx_list, schedule tx */ 475 first_rec = list_first_entry(&ctx->tx_list, 476 struct tls_rec, list); 477 if (rec == first_rec) { 478 /* Schedule the transmission */ 479 if (!test_and_set_bit(BIT_TX_SCHEDULED, 480 &ctx->tx_bitmask)) 481 schedule_delayed_work(&ctx->tx_work.work, 1); 482 } 483 } 484 485 spin_lock_bh(&ctx->encrypt_compl_lock); 486 pending = atomic_dec_return(&ctx->encrypt_pending); 487 488 if (!pending && ctx->async_notify) 489 complete(&ctx->async_wait.completion); 490 spin_unlock_bh(&ctx->encrypt_compl_lock); 491} 492 493static int tls_do_encryption(struct sock *sk, 494 struct tls_context *tls_ctx, 495 struct tls_sw_context_tx *ctx, 496 struct aead_request *aead_req, 497 size_t data_len, u32 start) 498{ 499 struct tls_prot_info *prot = &tls_ctx->prot_info; 500 struct tls_rec *rec = ctx->open_rec; 501 struct sk_msg *msg_en = &rec->msg_encrypted; 502 struct scatterlist *sge = sk_msg_elem(msg_en, start); 503 int rc, iv_offset = 0; 504 505 /* For CCM based ciphers, first byte of IV is a constant */ 506 if (prot->cipher_type == TLS_CIPHER_AES_CCM_128) { 507 rec->iv_data[0] = TLS_AES_CCM_IV_B0_BYTE; 508 iv_offset = 1; 509 } 510 511 memcpy(&rec->iv_data[iv_offset], tls_ctx->tx.iv, 512 prot->iv_size + prot->salt_size); 513 514 xor_iv_with_seq(prot->version, rec->iv_data + iv_offset, tls_ctx->tx.rec_seq); 515 516 sge->offset += prot->prepend_size; 517 sge->length -= prot->prepend_size; 518 519 msg_en->sg.curr = start; 520 521 aead_request_set_tfm(aead_req, ctx->aead_send); 522 aead_request_set_ad(aead_req, prot->aad_size); 523 aead_request_set_crypt(aead_req, rec->sg_aead_in, 524 rec->sg_aead_out, 525 data_len, rec->iv_data); 526 527 aead_request_set_callback(aead_req, CRYPTO_TFM_REQ_MAY_BACKLOG, 528 tls_encrypt_done, sk); 529 530 /* Add the record in tx_list */ 531 list_add_tail((struct list_head *)&rec->list, &ctx->tx_list); 532 atomic_inc(&ctx->encrypt_pending); 533 534 rc = crypto_aead_encrypt(aead_req); 535 if (!rc || rc != -EINPROGRESS) { 536 atomic_dec(&ctx->encrypt_pending); 537 sge->offset -= prot->prepend_size; 538 sge->length += prot->prepend_size; 539 } 540 541 if (!rc) { 542 WRITE_ONCE(rec->tx_ready, true); 543 } else if (rc != -EINPROGRESS) { 544 list_del(&rec->list); 545 return rc; 546 } 547 548 /* Unhook the record from context if encryption is not failure */ 549 ctx->open_rec = NULL; 550 tls_advance_record_sn(sk, prot, &tls_ctx->tx); 551 return rc; 552} 553 554static int tls_split_open_record(struct sock *sk, struct tls_rec *from, 555 struct tls_rec **to, struct sk_msg *msg_opl, 556 struct sk_msg *msg_oen, u32 split_point, 557 u32 tx_overhead_size, u32 *orig_end) 558{ 559 u32 i, j, bytes = 0, apply = msg_opl->apply_bytes; 560 struct scatterlist *sge, *osge, *nsge; 561 u32 orig_size = msg_opl->sg.size; 562 struct scatterlist tmp = { }; 563 struct sk_msg *msg_npl; 564 struct tls_rec *new; 565 int ret; 566 567 new = tls_get_rec(sk); 568 if (!new) 569 return -ENOMEM; 570 ret = sk_msg_alloc(sk, &new->msg_encrypted, msg_opl->sg.size + 571 tx_overhead_size, 0); 572 if (ret < 0) { 573 tls_free_rec(sk, new); 574 return ret; 575 } 576 577 *orig_end = msg_opl->sg.end; 578 i = msg_opl->sg.start; 579 sge = sk_msg_elem(msg_opl, i); 580 while (apply && sge->length) { 581 if (sge->length > apply) { 582 u32 len = sge->length - apply; 583 584 get_page(sg_page(sge)); 585 sg_set_page(&tmp, sg_page(sge), len, 586 sge->offset + apply); 587 sge->length = apply; 588 bytes += apply; 589 apply = 0; 590 } else { 591 apply -= sge->length; 592 bytes += sge->length; 593 } 594 595 sk_msg_iter_var_next(i); 596 if (i == msg_opl->sg.end) 597 break; 598 sge = sk_msg_elem(msg_opl, i); 599 } 600 601 msg_opl->sg.end = i; 602 msg_opl->sg.curr = i; 603 msg_opl->sg.copybreak = 0; 604 msg_opl->apply_bytes = 0; 605 msg_opl->sg.size = bytes; 606 607 msg_npl = &new->msg_plaintext; 608 msg_npl->apply_bytes = apply; 609 msg_npl->sg.size = orig_size - bytes; 610 611 j = msg_npl->sg.start; 612 nsge = sk_msg_elem(msg_npl, j); 613 if (tmp.length) { 614 memcpy(nsge, &tmp, sizeof(*nsge)); 615 sk_msg_iter_var_next(j); 616 nsge = sk_msg_elem(msg_npl, j); 617 } 618 619 osge = sk_msg_elem(msg_opl, i); 620 while (osge->length) { 621 memcpy(nsge, osge, sizeof(*nsge)); 622 sg_unmark_end(nsge); 623 sk_msg_iter_var_next(i); 624 sk_msg_iter_var_next(j); 625 if (i == *orig_end) 626 break; 627 osge = sk_msg_elem(msg_opl, i); 628 nsge = sk_msg_elem(msg_npl, j); 629 } 630 631 msg_npl->sg.end = j; 632 msg_npl->sg.curr = j; 633 msg_npl->sg.copybreak = 0; 634 635 *to = new; 636 return 0; 637} 638 639static void tls_merge_open_record(struct sock *sk, struct tls_rec *to, 640 struct tls_rec *from, u32 orig_end) 641{ 642 struct sk_msg *msg_npl = &from->msg_plaintext; 643 struct sk_msg *msg_opl = &to->msg_plaintext; 644 struct scatterlist *osge, *nsge; 645 u32 i, j; 646 647 i = msg_opl->sg.end; 648 sk_msg_iter_var_prev(i); 649 j = msg_npl->sg.start; 650 651 osge = sk_msg_elem(msg_opl, i); 652 nsge = sk_msg_elem(msg_npl, j); 653 654 if (sg_page(osge) == sg_page(nsge) && 655 osge->offset + osge->length == nsge->offset) { 656 osge->length += nsge->length; 657 put_page(sg_page(nsge)); 658 } 659 660 msg_opl->sg.end = orig_end; 661 msg_opl->sg.curr = orig_end; 662 msg_opl->sg.copybreak = 0; 663 msg_opl->apply_bytes = msg_opl->sg.size + msg_npl->sg.size; 664 msg_opl->sg.size += msg_npl->sg.size; 665 666 sk_msg_free(sk, &to->msg_encrypted); 667 sk_msg_xfer_full(&to->msg_encrypted, &from->msg_encrypted); 668 669 kfree(from); 670} 671 672static int tls_push_record(struct sock *sk, int flags, 673 unsigned char record_type) 674{ 675 struct tls_context *tls_ctx = tls_get_ctx(sk); 676 struct tls_prot_info *prot = &tls_ctx->prot_info; 677 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 678 struct tls_rec *rec = ctx->open_rec, *tmp = NULL; 679 u32 i, split_point, orig_end; 680 struct sk_msg *msg_pl, *msg_en; 681 struct aead_request *req; 682 bool split; 683 int rc; 684 685 if (!rec) 686 return 0; 687 688 msg_pl = &rec->msg_plaintext; 689 msg_en = &rec->msg_encrypted; 690 691 split_point = msg_pl->apply_bytes; 692 split = split_point && split_point < msg_pl->sg.size; 693 if (unlikely((!split && 694 msg_pl->sg.size + 695 prot->overhead_size > msg_en->sg.size) || 696 (split && 697 split_point + 698 prot->overhead_size > msg_en->sg.size))) { 699 split = true; 700 split_point = msg_en->sg.size; 701 } 702 if (split) { 703 rc = tls_split_open_record(sk, rec, &tmp, msg_pl, msg_en, 704 split_point, prot->overhead_size, 705 &orig_end); 706 if (rc < 0) 707 return rc; 708 /* This can happen if above tls_split_open_record allocates 709 * a single large encryption buffer instead of two smaller 710 * ones. In this case adjust pointers and continue without 711 * split. 712 */ 713 if (!msg_pl->sg.size) { 714 tls_merge_open_record(sk, rec, tmp, orig_end); 715 msg_pl = &rec->msg_plaintext; 716 msg_en = &rec->msg_encrypted; 717 split = false; 718 } 719 sk_msg_trim(sk, msg_en, msg_pl->sg.size + 720 prot->overhead_size); 721 } 722 723 rec->tx_flags = flags; 724 req = &rec->aead_req; 725 726 i = msg_pl->sg.end; 727 sk_msg_iter_var_prev(i); 728 729 rec->content_type = record_type; 730 if (prot->version == TLS_1_3_VERSION) { 731 /* Add content type to end of message. No padding added */ 732 sg_set_buf(&rec->sg_content_type, &rec->content_type, 1); 733 sg_mark_end(&rec->sg_content_type); 734 sg_chain(msg_pl->sg.data, msg_pl->sg.end + 1, 735 &rec->sg_content_type); 736 } else { 737 sg_mark_end(sk_msg_elem(msg_pl, i)); 738 } 739 740 if (msg_pl->sg.end < msg_pl->sg.start) { 741 sg_chain(&msg_pl->sg.data[msg_pl->sg.start], 742 MAX_SKB_FRAGS - msg_pl->sg.start + 1, 743 msg_pl->sg.data); 744 } 745 746 i = msg_pl->sg.start; 747 sg_chain(rec->sg_aead_in, 2, &msg_pl->sg.data[i]); 748 749 i = msg_en->sg.end; 750 sk_msg_iter_var_prev(i); 751 sg_mark_end(sk_msg_elem(msg_en, i)); 752 753 i = msg_en->sg.start; 754 sg_chain(rec->sg_aead_out, 2, &msg_en->sg.data[i]); 755 756 tls_make_aad(rec->aad_space, msg_pl->sg.size + prot->tail_size, 757 tls_ctx->tx.rec_seq, prot->rec_seq_size, 758 record_type, prot->version); 759 760 tls_fill_prepend(tls_ctx, 761 page_address(sg_page(&msg_en->sg.data[i])) + 762 msg_en->sg.data[i].offset, 763 msg_pl->sg.size + prot->tail_size, 764 record_type, prot->version); 765 766 tls_ctx->pending_open_record_frags = false; 767 768 rc = tls_do_encryption(sk, tls_ctx, ctx, req, 769 msg_pl->sg.size + prot->tail_size, i); 770 if (rc < 0) { 771 if (rc != -EINPROGRESS) { 772 tls_err_abort(sk, -EBADMSG); 773 if (split) { 774 tls_ctx->pending_open_record_frags = true; 775 tls_merge_open_record(sk, rec, tmp, orig_end); 776 } 777 } 778 ctx->async_capable = 1; 779 return rc; 780 } else if (split) { 781 msg_pl = &tmp->msg_plaintext; 782 msg_en = &tmp->msg_encrypted; 783 sk_msg_trim(sk, msg_en, msg_pl->sg.size + prot->overhead_size); 784 tls_ctx->pending_open_record_frags = true; 785 ctx->open_rec = tmp; 786 } 787 788 return tls_tx_records(sk, flags); 789} 790 791static int bpf_exec_tx_verdict(struct sk_msg *msg, struct sock *sk, 792 bool full_record, u8 record_type, 793 ssize_t *copied, int flags) 794{ 795 struct tls_context *tls_ctx = tls_get_ctx(sk); 796 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 797 struct sk_msg msg_redir = { }; 798 struct sk_psock *psock; 799 struct sock *sk_redir; 800 struct tls_rec *rec; 801 bool enospc, policy; 802 int err = 0, send; 803 u32 delta = 0; 804 805 policy = !(flags & MSG_SENDPAGE_NOPOLICY); 806 psock = sk_psock_get(sk); 807 if (!psock || !policy) { 808 err = tls_push_record(sk, flags, record_type); 809 if (err && err != -EINPROGRESS && sk->sk_err == EBADMSG) { 810 *copied -= sk_msg_free(sk, msg); 811 tls_free_open_rec(sk); 812 err = -sk->sk_err; 813 } 814 if (psock) 815 sk_psock_put(sk, psock); 816 return err; 817 } 818more_data: 819 enospc = sk_msg_full(msg); 820 if (psock->eval == __SK_NONE) { 821 delta = msg->sg.size; 822 psock->eval = sk_psock_msg_verdict(sk, psock, msg); 823 delta -= msg->sg.size; 824 } 825 if (msg->cork_bytes && msg->cork_bytes > msg->sg.size && 826 !enospc && !full_record) { 827 err = -ENOSPC; 828 goto out_err; 829 } 830 msg->cork_bytes = 0; 831 send = msg->sg.size; 832 if (msg->apply_bytes && msg->apply_bytes < send) 833 send = msg->apply_bytes; 834 835 switch (psock->eval) { 836 case __SK_PASS: 837 err = tls_push_record(sk, flags, record_type); 838 if (err && err != -EINPROGRESS && sk->sk_err == EBADMSG) { 839 *copied -= sk_msg_free(sk, msg); 840 tls_free_open_rec(sk); 841 err = -sk->sk_err; 842 goto out_err; 843 } 844 break; 845 case __SK_REDIRECT: 846 sk_redir = psock->sk_redir; 847 memcpy(&msg_redir, msg, sizeof(*msg)); 848 if (msg->apply_bytes < send) 849 msg->apply_bytes = 0; 850 else 851 msg->apply_bytes -= send; 852 sk_msg_return_zero(sk, msg, send); 853 msg->sg.size -= send; 854 release_sock(sk); 855 err = tcp_bpf_sendmsg_redir(sk_redir, &msg_redir, send, flags); 856 lock_sock(sk); 857 if (err < 0) { 858 *copied -= sk_msg_free_nocharge(sk, &msg_redir); 859 msg->sg.size = 0; 860 } 861 if (msg->sg.size == 0) 862 tls_free_open_rec(sk); 863 break; 864 case __SK_DROP: 865 default: 866 sk_msg_free_partial(sk, msg, send); 867 if (msg->apply_bytes < send) 868 msg->apply_bytes = 0; 869 else 870 msg->apply_bytes -= send; 871 if (msg->sg.size == 0) 872 tls_free_open_rec(sk); 873 *copied -= (send + delta); 874 err = -EACCES; 875 } 876 877 if (likely(!err)) { 878 bool reset_eval = !ctx->open_rec; 879 880 rec = ctx->open_rec; 881 if (rec) { 882 msg = &rec->msg_plaintext; 883 if (!msg->apply_bytes) 884 reset_eval = true; 885 } 886 if (reset_eval) { 887 psock->eval = __SK_NONE; 888 if (psock->sk_redir) { 889 sock_put(psock->sk_redir); 890 psock->sk_redir = NULL; 891 } 892 } 893 if (rec) 894 goto more_data; 895 } 896 out_err: 897 sk_psock_put(sk, psock); 898 return err; 899} 900 901static int tls_sw_push_pending_record(struct sock *sk, int flags) 902{ 903 struct tls_context *tls_ctx = tls_get_ctx(sk); 904 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 905 struct tls_rec *rec = ctx->open_rec; 906 struct sk_msg *msg_pl; 907 size_t copied; 908 909 if (!rec) 910 return 0; 911 912 msg_pl = &rec->msg_plaintext; 913 copied = msg_pl->sg.size; 914 if (!copied) 915 return 0; 916 917 return bpf_exec_tx_verdict(msg_pl, sk, true, TLS_RECORD_TYPE_DATA, 918 &copied, flags); 919} 920 921int tls_sw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size) 922{ 923 long timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT); 924 struct tls_context *tls_ctx = tls_get_ctx(sk); 925 struct tls_prot_info *prot = &tls_ctx->prot_info; 926 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 927 bool async_capable = ctx->async_capable; 928 unsigned char record_type = TLS_RECORD_TYPE_DATA; 929 bool is_kvec = iov_iter_is_kvec(&msg->msg_iter); 930 bool eor = !(msg->msg_flags & MSG_MORE); 931 size_t try_to_copy; 932 ssize_t copied = 0; 933 struct sk_msg *msg_pl, *msg_en; 934 struct tls_rec *rec; 935 int required_size; 936 int num_async = 0; 937 bool full_record; 938 int record_room; 939 int num_zc = 0; 940 int orig_size; 941 int ret = 0; 942 int pending; 943 944 if (msg->msg_flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL | 945 MSG_CMSG_COMPAT)) 946 return -EOPNOTSUPP; 947 948 ret = mutex_lock_interruptible(&tls_ctx->tx_lock); 949 if (ret) 950 return ret; 951 lock_sock(sk); 952 953 if (unlikely(msg->msg_controllen)) { 954 ret = tls_proccess_cmsg(sk, msg, &record_type); 955 if (ret) { 956 if (ret == -EINPROGRESS) 957 num_async++; 958 else if (ret != -EAGAIN) 959 goto send_end; 960 } 961 } 962 963 while (msg_data_left(msg)) { 964 if (sk->sk_err) { 965 ret = -sk->sk_err; 966 goto send_end; 967 } 968 969 if (ctx->open_rec) 970 rec = ctx->open_rec; 971 else 972 rec = ctx->open_rec = tls_get_rec(sk); 973 if (!rec) { 974 ret = -ENOMEM; 975 goto send_end; 976 } 977 978 msg_pl = &rec->msg_plaintext; 979 msg_en = &rec->msg_encrypted; 980 981 orig_size = msg_pl->sg.size; 982 full_record = false; 983 try_to_copy = msg_data_left(msg); 984 record_room = TLS_MAX_PAYLOAD_SIZE - msg_pl->sg.size; 985 if (try_to_copy >= record_room) { 986 try_to_copy = record_room; 987 full_record = true; 988 } 989 990 required_size = msg_pl->sg.size + try_to_copy + 991 prot->overhead_size; 992 993 if (!sk_stream_memory_free(sk)) 994 goto wait_for_sndbuf; 995 996alloc_encrypted: 997 ret = tls_alloc_encrypted_msg(sk, required_size); 998 if (ret) { 999 if (ret != -ENOSPC) 1000 goto wait_for_memory; 1001 1002 /* Adjust try_to_copy according to the amount that was 1003 * actually allocated. The difference is due 1004 * to max sg elements limit 1005 */ 1006 try_to_copy -= required_size - msg_en->sg.size; 1007 full_record = true; 1008 } 1009 1010 if (!is_kvec && (full_record || eor) && !async_capable) { 1011 u32 first = msg_pl->sg.end; 1012 1013 ret = sk_msg_zerocopy_from_iter(sk, &msg->msg_iter, 1014 msg_pl, try_to_copy); 1015 if (ret) 1016 goto fallback_to_reg_send; 1017 1018 num_zc++; 1019 copied += try_to_copy; 1020 1021 sk_msg_sg_copy_set(msg_pl, first); 1022 ret = bpf_exec_tx_verdict(msg_pl, sk, full_record, 1023 record_type, &copied, 1024 msg->msg_flags); 1025 if (ret) { 1026 if (ret == -EINPROGRESS) 1027 num_async++; 1028 else if (ret == -ENOMEM) 1029 goto wait_for_memory; 1030 else if (ctx->open_rec && ret == -ENOSPC) 1031 goto rollback_iter; 1032 else if (ret != -EAGAIN) 1033 goto send_end; 1034 } 1035 continue; 1036rollback_iter: 1037 copied -= try_to_copy; 1038 sk_msg_sg_copy_clear(msg_pl, first); 1039 iov_iter_revert(&msg->msg_iter, 1040 msg_pl->sg.size - orig_size); 1041fallback_to_reg_send: 1042 sk_msg_trim(sk, msg_pl, orig_size); 1043 } 1044 1045 required_size = msg_pl->sg.size + try_to_copy; 1046 1047 ret = tls_clone_plaintext_msg(sk, required_size); 1048 if (ret) { 1049 if (ret != -ENOSPC) 1050 goto send_end; 1051 1052 /* Adjust try_to_copy according to the amount that was 1053 * actually allocated. The difference is due 1054 * to max sg elements limit 1055 */ 1056 try_to_copy -= required_size - msg_pl->sg.size; 1057 full_record = true; 1058 sk_msg_trim(sk, msg_en, 1059 msg_pl->sg.size + prot->overhead_size); 1060 } 1061 1062 if (try_to_copy) { 1063 ret = sk_msg_memcopy_from_iter(sk, &msg->msg_iter, 1064 msg_pl, try_to_copy); 1065 if (ret < 0) 1066 goto trim_sgl; 1067 } 1068 1069 /* Open records defined only if successfully copied, otherwise 1070 * we would trim the sg but not reset the open record frags. 1071 */ 1072 tls_ctx->pending_open_record_frags = true; 1073 copied += try_to_copy; 1074 if (full_record || eor) { 1075 ret = bpf_exec_tx_verdict(msg_pl, sk, full_record, 1076 record_type, &copied, 1077 msg->msg_flags); 1078 if (ret) { 1079 if (ret == -EINPROGRESS) 1080 num_async++; 1081 else if (ret == -ENOMEM) 1082 goto wait_for_memory; 1083 else if (ret != -EAGAIN) { 1084 if (ret == -ENOSPC) 1085 ret = 0; 1086 goto send_end; 1087 } 1088 } 1089 } 1090 1091 continue; 1092 1093wait_for_sndbuf: 1094 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 1095wait_for_memory: 1096 ret = sk_stream_wait_memory(sk, &timeo); 1097 if (ret) { 1098trim_sgl: 1099 if (ctx->open_rec) 1100 tls_trim_both_msgs(sk, orig_size); 1101 goto send_end; 1102 } 1103 1104 if (ctx->open_rec && msg_en->sg.size < required_size) 1105 goto alloc_encrypted; 1106 } 1107 1108 if (!num_async) { 1109 goto send_end; 1110 } else if (num_zc) { 1111 /* Wait for pending encryptions to get completed */ 1112 spin_lock_bh(&ctx->encrypt_compl_lock); 1113 ctx->async_notify = true; 1114 1115 pending = atomic_read(&ctx->encrypt_pending); 1116 spin_unlock_bh(&ctx->encrypt_compl_lock); 1117 if (pending) 1118 crypto_wait_req(-EINPROGRESS, &ctx->async_wait); 1119 else 1120 reinit_completion(&ctx->async_wait.completion); 1121 1122 /* There can be no concurrent accesses, since we have no 1123 * pending encrypt operations 1124 */ 1125 WRITE_ONCE(ctx->async_notify, false); 1126 1127 if (ctx->async_wait.err) { 1128 ret = ctx->async_wait.err; 1129 copied = 0; 1130 } 1131 } 1132 1133 /* Transmit if any encryptions have completed */ 1134 if (test_and_clear_bit(BIT_TX_SCHEDULED, &ctx->tx_bitmask)) { 1135 cancel_delayed_work(&ctx->tx_work.work); 1136 tls_tx_records(sk, msg->msg_flags); 1137 } 1138 1139send_end: 1140 ret = sk_stream_error(sk, msg->msg_flags, ret); 1141 1142 release_sock(sk); 1143 mutex_unlock(&tls_ctx->tx_lock); 1144 return copied > 0 ? copied : ret; 1145} 1146 1147static int tls_sw_do_sendpage(struct sock *sk, struct page *page, 1148 int offset, size_t size, int flags) 1149{ 1150 long timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT); 1151 struct tls_context *tls_ctx = tls_get_ctx(sk); 1152 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 1153 struct tls_prot_info *prot = &tls_ctx->prot_info; 1154 unsigned char record_type = TLS_RECORD_TYPE_DATA; 1155 struct sk_msg *msg_pl; 1156 struct tls_rec *rec; 1157 int num_async = 0; 1158 ssize_t copied = 0; 1159 bool full_record; 1160 int record_room; 1161 int ret = 0; 1162 bool eor; 1163 1164 eor = !(flags & MSG_SENDPAGE_NOTLAST); 1165 sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk); 1166 1167 /* Call the sk_stream functions to manage the sndbuf mem. */ 1168 while (size > 0) { 1169 size_t copy, required_size; 1170 1171 if (sk->sk_err) { 1172 ret = -sk->sk_err; 1173 goto sendpage_end; 1174 } 1175 1176 if (ctx->open_rec) 1177 rec = ctx->open_rec; 1178 else 1179 rec = ctx->open_rec = tls_get_rec(sk); 1180 if (!rec) { 1181 ret = -ENOMEM; 1182 goto sendpage_end; 1183 } 1184 1185 msg_pl = &rec->msg_plaintext; 1186 1187 full_record = false; 1188 record_room = TLS_MAX_PAYLOAD_SIZE - msg_pl->sg.size; 1189 copy = size; 1190 if (copy >= record_room) { 1191 copy = record_room; 1192 full_record = true; 1193 } 1194 1195 required_size = msg_pl->sg.size + copy + prot->overhead_size; 1196 1197 if (!sk_stream_memory_free(sk)) 1198 goto wait_for_sndbuf; 1199alloc_payload: 1200 ret = tls_alloc_encrypted_msg(sk, required_size); 1201 if (ret) { 1202 if (ret != -ENOSPC) 1203 goto wait_for_memory; 1204 1205 /* Adjust copy according to the amount that was 1206 * actually allocated. The difference is due 1207 * to max sg elements limit 1208 */ 1209 copy -= required_size - msg_pl->sg.size; 1210 full_record = true; 1211 } 1212 1213 sk_msg_page_add(msg_pl, page, copy, offset); 1214 msg_pl->sg.copybreak = 0; 1215 msg_pl->sg.curr = msg_pl->sg.end; 1216 sk_mem_charge(sk, copy); 1217 1218 offset += copy; 1219 size -= copy; 1220 copied += copy; 1221 1222 tls_ctx->pending_open_record_frags = true; 1223 if (full_record || eor || sk_msg_full(msg_pl)) { 1224 ret = bpf_exec_tx_verdict(msg_pl, sk, full_record, 1225 record_type, &copied, flags); 1226 if (ret) { 1227 if (ret == -EINPROGRESS) 1228 num_async++; 1229 else if (ret == -ENOMEM) 1230 goto wait_for_memory; 1231 else if (ret != -EAGAIN) { 1232 if (ret == -ENOSPC) 1233 ret = 0; 1234 goto sendpage_end; 1235 } 1236 } 1237 } 1238 continue; 1239wait_for_sndbuf: 1240 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 1241wait_for_memory: 1242 ret = sk_stream_wait_memory(sk, &timeo); 1243 if (ret) { 1244 if (ctx->open_rec) 1245 tls_trim_both_msgs(sk, msg_pl->sg.size); 1246 goto sendpage_end; 1247 } 1248 1249 if (ctx->open_rec) 1250 goto alloc_payload; 1251 } 1252 1253 if (num_async) { 1254 /* Transmit if any encryptions have completed */ 1255 if (test_and_clear_bit(BIT_TX_SCHEDULED, &ctx->tx_bitmask)) { 1256 cancel_delayed_work(&ctx->tx_work.work); 1257 tls_tx_records(sk, flags); 1258 } 1259 } 1260sendpage_end: 1261 ret = sk_stream_error(sk, flags, ret); 1262 return copied > 0 ? copied : ret; 1263} 1264 1265int tls_sw_sendpage_locked(struct sock *sk, struct page *page, 1266 int offset, size_t size, int flags) 1267{ 1268 if (flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL | 1269 MSG_SENDPAGE_NOTLAST | MSG_SENDPAGE_NOPOLICY | 1270 MSG_NO_SHARED_FRAGS)) 1271 return -EOPNOTSUPP; 1272 1273 return tls_sw_do_sendpage(sk, page, offset, size, flags); 1274} 1275 1276int tls_sw_sendpage(struct sock *sk, struct page *page, 1277 int offset, size_t size, int flags) 1278{ 1279 struct tls_context *tls_ctx = tls_get_ctx(sk); 1280 int ret; 1281 1282 if (flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL | 1283 MSG_SENDPAGE_NOTLAST | MSG_SENDPAGE_NOPOLICY)) 1284 return -EOPNOTSUPP; 1285 1286 ret = mutex_lock_interruptible(&tls_ctx->tx_lock); 1287 if (ret) 1288 return ret; 1289 lock_sock(sk); 1290 ret = tls_sw_do_sendpage(sk, page, offset, size, flags); 1291 release_sock(sk); 1292 mutex_unlock(&tls_ctx->tx_lock); 1293 return ret; 1294} 1295 1296static struct sk_buff *tls_wait_data(struct sock *sk, struct sk_psock *psock, 1297 bool nonblock, long timeo, int *err) 1298{ 1299 struct tls_context *tls_ctx = tls_get_ctx(sk); 1300 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 1301 struct sk_buff *skb; 1302 DEFINE_WAIT_FUNC(wait, woken_wake_function); 1303 1304 while (!(skb = ctx->recv_pkt) && sk_psock_queue_empty(psock)) { 1305 if (sk->sk_err) { 1306 *err = sock_error(sk); 1307 return NULL; 1308 } 1309 1310 if (!skb_queue_empty(&sk->sk_receive_queue)) { 1311 __strp_unpause(&ctx->strp); 1312 if (ctx->recv_pkt) 1313 return ctx->recv_pkt; 1314 } 1315 1316 if (sk->sk_shutdown & RCV_SHUTDOWN) 1317 return NULL; 1318 1319 if (sock_flag(sk, SOCK_DONE)) 1320 return NULL; 1321 1322 if (nonblock || !timeo) { 1323 *err = -EAGAIN; 1324 return NULL; 1325 } 1326 1327 add_wait_queue(sk_sleep(sk), &wait); 1328 sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk); 1329 sk_wait_event(sk, &timeo, 1330 ctx->recv_pkt != skb || 1331 !sk_psock_queue_empty(psock), 1332 &wait); 1333 sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk); 1334 remove_wait_queue(sk_sleep(sk), &wait); 1335 1336 /* Handle signals */ 1337 if (signal_pending(current)) { 1338 *err = sock_intr_errno(timeo); 1339 return NULL; 1340 } 1341 } 1342 1343 return skb; 1344} 1345 1346static int tls_setup_from_iter(struct sock *sk, struct iov_iter *from, 1347 int length, int *pages_used, 1348 unsigned int *size_used, 1349 struct scatterlist *to, 1350 int to_max_pages) 1351{ 1352 int rc = 0, i = 0, num_elem = *pages_used, maxpages; 1353 struct page *pages[MAX_SKB_FRAGS]; 1354 unsigned int size = *size_used; 1355 ssize_t copied, use; 1356 size_t offset; 1357 1358 while (length > 0) { 1359 i = 0; 1360 maxpages = to_max_pages - num_elem; 1361 if (maxpages == 0) { 1362 rc = -EFAULT; 1363 goto out; 1364 } 1365 copied = iov_iter_get_pages(from, pages, 1366 length, 1367 maxpages, &offset); 1368 if (copied <= 0) { 1369 rc = -EFAULT; 1370 goto out; 1371 } 1372 1373 iov_iter_advance(from, copied); 1374 1375 length -= copied; 1376 size += copied; 1377 while (copied) { 1378 use = min_t(int, copied, PAGE_SIZE - offset); 1379 1380 sg_set_page(&to[num_elem], 1381 pages[i], use, offset); 1382 sg_unmark_end(&to[num_elem]); 1383 /* We do not uncharge memory from this API */ 1384 1385 offset = 0; 1386 copied -= use; 1387 1388 i++; 1389 num_elem++; 1390 } 1391 } 1392 /* Mark the end in the last sg entry if newly added */ 1393 if (num_elem > *pages_used) 1394 sg_mark_end(&to[num_elem - 1]); 1395out: 1396 if (rc) 1397 iov_iter_revert(from, size - *size_used); 1398 *size_used = size; 1399 *pages_used = num_elem; 1400 1401 return rc; 1402} 1403 1404/* This function decrypts the input skb into either out_iov or in out_sg 1405 * or in skb buffers itself. The input parameter 'zc' indicates if 1406 * zero-copy mode needs to be tried or not. With zero-copy mode, either 1407 * out_iov or out_sg must be non-NULL. In case both out_iov and out_sg are 1408 * NULL, then the decryption happens inside skb buffers itself, i.e. 1409 * zero-copy gets disabled and 'zc' is updated. 1410 */ 1411 1412static int decrypt_internal(struct sock *sk, struct sk_buff *skb, 1413 struct iov_iter *out_iov, 1414 struct scatterlist *out_sg, 1415 int *chunk, bool *zc, bool async) 1416{ 1417 struct tls_context *tls_ctx = tls_get_ctx(sk); 1418 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 1419 struct tls_prot_info *prot = &tls_ctx->prot_info; 1420 struct strp_msg *rxm = strp_msg(skb); 1421 int n_sgin, n_sgout, nsg, mem_size, aead_size, err, pages = 0; 1422 struct aead_request *aead_req; 1423 struct sk_buff *unused; 1424 u8 *aad, *iv, *mem = NULL; 1425 struct scatterlist *sgin = NULL; 1426 struct scatterlist *sgout = NULL; 1427 const int data_len = rxm->full_len - prot->overhead_size + 1428 prot->tail_size; 1429 int iv_offset = 0; 1430 1431 if (*zc && (out_iov || out_sg)) { 1432 if (out_iov) 1433 n_sgout = iov_iter_npages(out_iov, INT_MAX) + 1; 1434 else 1435 n_sgout = sg_nents(out_sg); 1436 n_sgin = skb_nsg(skb, rxm->offset + prot->prepend_size, 1437 rxm->full_len - prot->prepend_size); 1438 } else { 1439 n_sgout = 0; 1440 *zc = false; 1441 n_sgin = skb_cow_data(skb, 0, &unused); 1442 } 1443 1444 if (n_sgin < 1) 1445 return -EBADMSG; 1446 1447 /* Increment to accommodate AAD */ 1448 n_sgin = n_sgin + 1; 1449 1450 nsg = n_sgin + n_sgout; 1451 1452 aead_size = sizeof(*aead_req) + crypto_aead_reqsize(ctx->aead_recv); 1453 mem_size = aead_size + (nsg * sizeof(struct scatterlist)); 1454 mem_size = mem_size + prot->aad_size; 1455 mem_size = mem_size + crypto_aead_ivsize(ctx->aead_recv); 1456 1457 /* Allocate a single block of memory which contains 1458 * aead_req || sgin[] || sgout[] || aad || iv. 1459 * This order achieves correct alignment for aead_req, sgin, sgout. 1460 */ 1461 mem = kmalloc(mem_size, sk->sk_allocation); 1462 if (!mem) 1463 return -ENOMEM; 1464 1465 /* Segment the allocated memory */ 1466 aead_req = (struct aead_request *)mem; 1467 sgin = (struct scatterlist *)(mem + aead_size); 1468 sgout = sgin + n_sgin; 1469 aad = (u8 *)(sgout + n_sgout); 1470 iv = aad + prot->aad_size; 1471 1472 /* For CCM based ciphers, first byte of nonce+iv is always '2' */ 1473 if (prot->cipher_type == TLS_CIPHER_AES_CCM_128) { 1474 iv[0] = 2; 1475 iv_offset = 1; 1476 } 1477 1478 /* Prepare IV */ 1479 err = skb_copy_bits(skb, rxm->offset + TLS_HEADER_SIZE, 1480 iv + iv_offset + prot->salt_size, 1481 prot->iv_size); 1482 if (err < 0) { 1483 kfree(mem); 1484 return err; 1485 } 1486 if (prot->version == TLS_1_3_VERSION) 1487 memcpy(iv + iv_offset, tls_ctx->rx.iv, 1488 prot->iv_size + prot->salt_size); 1489 else 1490 memcpy(iv + iv_offset, tls_ctx->rx.iv, prot->salt_size); 1491 1492 xor_iv_with_seq(prot->version, iv + iv_offset, tls_ctx->rx.rec_seq); 1493 1494 /* Prepare AAD */ 1495 tls_make_aad(aad, rxm->full_len - prot->overhead_size + 1496 prot->tail_size, 1497 tls_ctx->rx.rec_seq, prot->rec_seq_size, 1498 ctx->control, prot->version); 1499 1500 /* Prepare sgin */ 1501 sg_init_table(sgin, n_sgin); 1502 sg_set_buf(&sgin[0], aad, prot->aad_size); 1503 err = skb_to_sgvec(skb, &sgin[1], 1504 rxm->offset + prot->prepend_size, 1505 rxm->full_len - prot->prepend_size); 1506 if (err < 0) { 1507 kfree(mem); 1508 return err; 1509 } 1510 1511 if (n_sgout) { 1512 if (out_iov) { 1513 sg_init_table(sgout, n_sgout); 1514 sg_set_buf(&sgout[0], aad, prot->aad_size); 1515 1516 *chunk = 0; 1517 err = tls_setup_from_iter(sk, out_iov, data_len, 1518 &pages, chunk, &sgout[1], 1519 (n_sgout - 1)); 1520 if (err < 0) 1521 goto fallback_to_reg_recv; 1522 } else if (out_sg) { 1523 memcpy(sgout, out_sg, n_sgout * sizeof(*sgout)); 1524 } else { 1525 goto fallback_to_reg_recv; 1526 } 1527 } else { 1528fallback_to_reg_recv: 1529 sgout = sgin; 1530 pages = 0; 1531 *chunk = data_len; 1532 *zc = false; 1533 } 1534 1535 /* Prepare and submit AEAD request */ 1536 err = tls_do_decryption(sk, skb, sgin, sgout, iv, 1537 data_len, aead_req, async); 1538 if (err == -EINPROGRESS) 1539 return err; 1540 1541 /* Release the pages in case iov was mapped to pages */ 1542 for (; pages > 0; pages--) 1543 put_page(sg_page(&sgout[pages])); 1544 1545 kfree(mem); 1546 return err; 1547} 1548 1549static int decrypt_skb_update(struct sock *sk, struct sk_buff *skb, 1550 struct iov_iter *dest, int *chunk, bool *zc, 1551 bool async) 1552{ 1553 struct tls_context *tls_ctx = tls_get_ctx(sk); 1554 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 1555 struct tls_prot_info *prot = &tls_ctx->prot_info; 1556 struct strp_msg *rxm = strp_msg(skb); 1557 int pad, err = 0; 1558 1559 if (!ctx->decrypted) { 1560 if (tls_ctx->rx_conf == TLS_HW) { 1561 err = tls_device_decrypted(sk, tls_ctx, skb, rxm); 1562 if (err < 0) 1563 return err; 1564 } 1565 1566 /* Still not decrypted after tls_device */ 1567 if (!ctx->decrypted) { 1568 err = decrypt_internal(sk, skb, dest, NULL, chunk, zc, 1569 async); 1570 if (err < 0) { 1571 if (err == -EINPROGRESS) 1572 tls_advance_record_sn(sk, prot, 1573 &tls_ctx->rx); 1574 else if (err == -EBADMSG) 1575 TLS_INC_STATS(sock_net(sk), 1576 LINUX_MIB_TLSDECRYPTERROR); 1577 return err; 1578 } 1579 } else { 1580 *zc = false; 1581 } 1582 1583 pad = padding_length(ctx, prot, skb); 1584 if (pad < 0) 1585 return pad; 1586 1587 rxm->full_len -= pad; 1588 rxm->offset += prot->prepend_size; 1589 rxm->full_len -= prot->overhead_size; 1590 tls_advance_record_sn(sk, prot, &tls_ctx->rx); 1591 ctx->decrypted = 1; 1592 ctx->saved_data_ready(sk); 1593 } else { 1594 *zc = false; 1595 } 1596 1597 return err; 1598} 1599 1600int decrypt_skb(struct sock *sk, struct sk_buff *skb, 1601 struct scatterlist *sgout) 1602{ 1603 bool zc = true; 1604 int chunk; 1605 1606 return decrypt_internal(sk, skb, NULL, sgout, &chunk, &zc, false); 1607} 1608 1609static bool tls_sw_advance_skb(struct sock *sk, struct sk_buff *skb, 1610 unsigned int len) 1611{ 1612 struct tls_context *tls_ctx = tls_get_ctx(sk); 1613 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 1614 1615 if (skb) { 1616 struct strp_msg *rxm = strp_msg(skb); 1617 1618 if (len < rxm->full_len) { 1619 rxm->offset += len; 1620 rxm->full_len -= len; 1621 return false; 1622 } 1623 consume_skb(skb); 1624 } 1625 1626 /* Finished with message */ 1627 ctx->recv_pkt = NULL; 1628 __strp_unpause(&ctx->strp); 1629 1630 return true; 1631} 1632 1633/* This function traverses the rx_list in tls receive context to copies the 1634 * decrypted records into the buffer provided by caller zero copy is not 1635 * true. Further, the records are removed from the rx_list if it is not a peek 1636 * case and the record has been consumed completely. 1637 */ 1638static int process_rx_list(struct tls_sw_context_rx *ctx, 1639 struct msghdr *msg, 1640 u8 *control, 1641 bool *cmsg, 1642 size_t skip, 1643 size_t len, 1644 bool zc, 1645 bool is_peek) 1646{ 1647 struct sk_buff *skb = skb_peek(&ctx->rx_list); 1648 u8 ctrl = *control; 1649 u8 msgc = *cmsg; 1650 struct tls_msg *tlm; 1651 ssize_t copied = 0; 1652 1653 /* Set the record type in 'control' if caller didn't pass it */ 1654 if (!ctrl && skb) { 1655 tlm = tls_msg(skb); 1656 ctrl = tlm->control; 1657 } 1658 1659 while (skip && skb) { 1660 struct strp_msg *rxm = strp_msg(skb); 1661 tlm = tls_msg(skb); 1662 1663 /* Cannot process a record of different type */ 1664 if (ctrl != tlm->control) 1665 return 0; 1666 1667 if (skip < rxm->full_len) 1668 break; 1669 1670 skip = skip - rxm->full_len; 1671 skb = skb_peek_next(skb, &ctx->rx_list); 1672 } 1673 1674 while (len && skb) { 1675 struct sk_buff *next_skb; 1676 struct strp_msg *rxm = strp_msg(skb); 1677 int chunk = min_t(unsigned int, rxm->full_len - skip, len); 1678 1679 tlm = tls_msg(skb); 1680 1681 /* Cannot process a record of different type */ 1682 if (ctrl != tlm->control) 1683 return 0; 1684 1685 /* Set record type if not already done. For a non-data record, 1686 * do not proceed if record type could not be copied. 1687 */ 1688 if (!msgc) { 1689 int cerr = put_cmsg(msg, SOL_TLS, TLS_GET_RECORD_TYPE, 1690 sizeof(ctrl), &ctrl); 1691 msgc = true; 1692 if (ctrl != TLS_RECORD_TYPE_DATA) { 1693 if (cerr || msg->msg_flags & MSG_CTRUNC) 1694 return -EIO; 1695 1696 *cmsg = msgc; 1697 } 1698 } 1699 1700 if (!zc || (rxm->full_len - skip) > len) { 1701 int err = skb_copy_datagram_msg(skb, rxm->offset + skip, 1702 msg, chunk); 1703 if (err < 0) 1704 return err; 1705 } 1706 1707 len = len - chunk; 1708 copied = copied + chunk; 1709 1710 /* Consume the data from record if it is non-peek case*/ 1711 if (!is_peek) { 1712 rxm->offset = rxm->offset + chunk; 1713 rxm->full_len = rxm->full_len - chunk; 1714 1715 /* Return if there is unconsumed data in the record */ 1716 if (rxm->full_len - skip) 1717 break; 1718 } 1719 1720 /* The remaining skip-bytes must lie in 1st record in rx_list. 1721 * So from the 2nd record, 'skip' should be 0. 1722 */ 1723 skip = 0; 1724 1725 if (msg) 1726 msg->msg_flags |= MSG_EOR; 1727 1728 next_skb = skb_peek_next(skb, &ctx->rx_list); 1729 1730 if (!is_peek) { 1731 skb_unlink(skb, &ctx->rx_list); 1732 consume_skb(skb); 1733 } 1734 1735 skb = next_skb; 1736 } 1737 1738 *control = ctrl; 1739 return copied; 1740} 1741 1742int tls_sw_recvmsg(struct sock *sk, 1743 struct msghdr *msg, 1744 size_t len, 1745 int nonblock, 1746 int flags, 1747 int *addr_len) 1748{ 1749 struct tls_context *tls_ctx = tls_get_ctx(sk); 1750 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 1751 struct tls_prot_info *prot = &tls_ctx->prot_info; 1752 struct sk_psock *psock; 1753 unsigned char control = 0; 1754 ssize_t decrypted = 0; 1755 struct strp_msg *rxm; 1756 struct tls_msg *tlm; 1757 struct sk_buff *skb; 1758 ssize_t copied = 0; 1759 bool cmsg = false; 1760 int target, err = 0; 1761 long timeo; 1762 bool is_kvec = iov_iter_is_kvec(&msg->msg_iter); 1763 bool is_peek = flags & MSG_PEEK; 1764 bool bpf_strp_enabled; 1765 int num_async = 0; 1766 int pending; 1767 1768 flags |= nonblock; 1769 1770 if (unlikely(flags & MSG_ERRQUEUE)) 1771 return sock_recv_errqueue(sk, msg, len, SOL_IP, IP_RECVERR); 1772 1773 psock = sk_psock_get(sk); 1774 lock_sock(sk); 1775 bpf_strp_enabled = sk_psock_strp_enabled(psock); 1776 1777 /* Process pending decrypted records. It must be non-zero-copy */ 1778 err = process_rx_list(ctx, msg, &control, &cmsg, 0, len, false, 1779 is_peek); 1780 if (err < 0) { 1781 tls_err_abort(sk, err); 1782 goto end; 1783 } else { 1784 copied = err; 1785 } 1786 1787 if (len <= copied) 1788 goto recv_end; 1789 1790 target = sock_rcvlowat(sk, flags & MSG_WAITALL, len); 1791 len = len - copied; 1792 timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT); 1793 1794 while (len && (decrypted + copied < target || ctx->recv_pkt)) { 1795 bool retain_skb = false; 1796 bool zc = false; 1797 int to_decrypt; 1798 int chunk = 0; 1799 bool async_capable; 1800 bool async = false; 1801 1802 skb = tls_wait_data(sk, psock, flags & MSG_DONTWAIT, timeo, &err); 1803 if (!skb) { 1804 if (psock) { 1805 int ret = __tcp_bpf_recvmsg(sk, psock, 1806 msg, len, flags); 1807 1808 if (ret > 0) { 1809 decrypted += ret; 1810 len -= ret; 1811 continue; 1812 } 1813 } 1814 goto recv_end; 1815 } else { 1816 tlm = tls_msg(skb); 1817 if (prot->version == TLS_1_3_VERSION) 1818 tlm->control = 0; 1819 else 1820 tlm->control = ctx->control; 1821 } 1822 1823 rxm = strp_msg(skb); 1824 1825 to_decrypt = rxm->full_len - prot->overhead_size; 1826 1827 if (to_decrypt <= len && !is_kvec && !is_peek && 1828 ctx->control == TLS_RECORD_TYPE_DATA && 1829 prot->version != TLS_1_3_VERSION && 1830 !bpf_strp_enabled) 1831 zc = true; 1832 1833 /* Do not use async mode if record is non-data */ 1834 if (ctx->control == TLS_RECORD_TYPE_DATA && !bpf_strp_enabled) 1835 async_capable = ctx->async_capable; 1836 else 1837 async_capable = false; 1838 1839 err = decrypt_skb_update(sk, skb, &msg->msg_iter, 1840 &chunk, &zc, async_capable); 1841 if (err < 0 && err != -EINPROGRESS) { 1842 tls_err_abort(sk, -EBADMSG); 1843 goto recv_end; 1844 } 1845 1846 if (err == -EINPROGRESS) { 1847 async = true; 1848 num_async++; 1849 } else if (prot->version == TLS_1_3_VERSION) { 1850 tlm->control = ctx->control; 1851 } 1852 1853 /* If the type of records being processed is not known yet, 1854 * set it to record type just dequeued. If it is already known, 1855 * but does not match the record type just dequeued, go to end. 1856 * We always get record type here since for tls1.2, record type 1857 * is known just after record is dequeued from stream parser. 1858 * For tls1.3, we disable async. 1859 */ 1860 1861 if (!control) 1862 control = tlm->control; 1863 else if (control != tlm->control) 1864 goto recv_end; 1865 1866 if (!cmsg) { 1867 int cerr; 1868 1869 cerr = put_cmsg(msg, SOL_TLS, TLS_GET_RECORD_TYPE, 1870 sizeof(control), &control); 1871 cmsg = true; 1872 if (control != TLS_RECORD_TYPE_DATA) { 1873 if (cerr || msg->msg_flags & MSG_CTRUNC) { 1874 err = -EIO; 1875 goto recv_end; 1876 } 1877 } 1878 } 1879 1880 if (async) 1881 goto pick_next_record; 1882 1883 if (!zc) { 1884 if (bpf_strp_enabled) { 1885 err = sk_psock_tls_strp_read(psock, skb); 1886 if (err != __SK_PASS) { 1887 rxm->offset = rxm->offset + rxm->full_len; 1888 rxm->full_len = 0; 1889 if (err == __SK_DROP) 1890 consume_skb(skb); 1891 ctx->recv_pkt = NULL; 1892 __strp_unpause(&ctx->strp); 1893 continue; 1894 } 1895 } 1896 1897 if (rxm->full_len > len) { 1898 retain_skb = true; 1899 chunk = len; 1900 } else { 1901 chunk = rxm->full_len; 1902 } 1903 1904 err = skb_copy_datagram_msg(skb, rxm->offset, 1905 msg, chunk); 1906 if (err < 0) 1907 goto recv_end; 1908 1909 if (!is_peek) { 1910 rxm->offset = rxm->offset + chunk; 1911 rxm->full_len = rxm->full_len - chunk; 1912 } 1913 } 1914 1915pick_next_record: 1916 if (chunk > len) 1917 chunk = len; 1918 1919 decrypted += chunk; 1920 len -= chunk; 1921 1922 /* For async or peek case, queue the current skb */ 1923 if (async || is_peek || retain_skb) { 1924 skb_queue_tail(&ctx->rx_list, skb); 1925 skb = NULL; 1926 } 1927 1928 if (tls_sw_advance_skb(sk, skb, chunk)) { 1929 /* Return full control message to 1930 * userspace before trying to parse 1931 * another message type 1932 */ 1933 msg->msg_flags |= MSG_EOR; 1934 if (control != TLS_RECORD_TYPE_DATA) 1935 goto recv_end; 1936 } else { 1937 break; 1938 } 1939 } 1940 1941recv_end: 1942 if (num_async) { 1943 /* Wait for all previously submitted records to be decrypted */ 1944 spin_lock_bh(&ctx->decrypt_compl_lock); 1945 ctx->async_notify = true; 1946 pending = atomic_read(&ctx->decrypt_pending); 1947 spin_unlock_bh(&ctx->decrypt_compl_lock); 1948 if (pending) { 1949 err = crypto_wait_req(-EINPROGRESS, &ctx->async_wait); 1950 if (err) { 1951 /* one of async decrypt failed */ 1952 tls_err_abort(sk, err); 1953 copied = 0; 1954 decrypted = 0; 1955 goto end; 1956 } 1957 } else { 1958 reinit_completion(&ctx->async_wait.completion); 1959 } 1960 1961 /* There can be no concurrent accesses, since we have no 1962 * pending decrypt operations 1963 */ 1964 WRITE_ONCE(ctx->async_notify, false); 1965 1966 /* Drain records from the rx_list & copy if required */ 1967 if (is_peek || is_kvec) 1968 err = process_rx_list(ctx, msg, &control, &cmsg, copied, 1969 decrypted, false, is_peek); 1970 else 1971 err = process_rx_list(ctx, msg, &control, &cmsg, 0, 1972 decrypted, true, is_peek); 1973 if (err < 0) { 1974 tls_err_abort(sk, err); 1975 copied = 0; 1976 goto end; 1977 } 1978 } 1979 1980 copied += decrypted; 1981 1982end: 1983 release_sock(sk); 1984 if (psock) 1985 sk_psock_put(sk, psock); 1986 return copied ? : err; 1987} 1988 1989ssize_t tls_sw_splice_read(struct socket *sock, loff_t *ppos, 1990 struct pipe_inode_info *pipe, 1991 size_t len, unsigned int flags) 1992{ 1993 struct tls_context *tls_ctx = tls_get_ctx(sock->sk); 1994 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 1995 struct strp_msg *rxm = NULL; 1996 struct sock *sk = sock->sk; 1997 struct sk_buff *skb; 1998 ssize_t copied = 0; 1999 int err = 0; 2000 long timeo; 2001 int chunk; 2002 bool zc = false; 2003 2004 lock_sock(sk); 2005 2006 timeo = sock_rcvtimeo(sk, flags & SPLICE_F_NONBLOCK); 2007 2008 skb = tls_wait_data(sk, NULL, flags & SPLICE_F_NONBLOCK, timeo, &err); 2009 if (!skb) 2010 goto splice_read_end; 2011 2012 err = decrypt_skb_update(sk, skb, NULL, &chunk, &zc, false); 2013 if (err < 0) { 2014 tls_err_abort(sk, -EBADMSG); 2015 goto splice_read_end; 2016 } 2017 2018 /* splice does not support reading control messages */ 2019 if (ctx->control != TLS_RECORD_TYPE_DATA) { 2020 err = -EINVAL; 2021 goto splice_read_end; 2022 } 2023 2024 rxm = strp_msg(skb); 2025 2026 chunk = min_t(unsigned int, rxm->full_len, len); 2027 copied = skb_splice_bits(skb, sk, rxm->offset, pipe, chunk, flags); 2028 if (copied < 0) 2029 goto splice_read_end; 2030 2031 if (likely(!(flags & MSG_PEEK))) 2032 tls_sw_advance_skb(sk, skb, copied); 2033 2034splice_read_end: 2035 release_sock(sk); 2036 return copied ? : err; 2037} 2038 2039bool tls_sw_stream_read(const struct sock *sk) 2040{ 2041 struct tls_context *tls_ctx = tls_get_ctx(sk); 2042 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 2043 bool ingress_empty = true; 2044 struct sk_psock *psock; 2045 2046 rcu_read_lock(); 2047 psock = sk_psock(sk); 2048 if (psock) 2049 ingress_empty = list_empty(&psock->ingress_msg); 2050 rcu_read_unlock(); 2051 2052 return !ingress_empty || ctx->recv_pkt || 2053 !skb_queue_empty(&ctx->rx_list); 2054} 2055 2056static int tls_read_size(struct strparser *strp, struct sk_buff *skb) 2057{ 2058 struct tls_context *tls_ctx = tls_get_ctx(strp->sk); 2059 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 2060 struct tls_prot_info *prot = &tls_ctx->prot_info; 2061 char header[TLS_HEADER_SIZE + MAX_IV_SIZE]; 2062 struct strp_msg *rxm = strp_msg(skb); 2063 size_t cipher_overhead; 2064 size_t data_len = 0; 2065 int ret; 2066 2067 /* Verify that we have a full TLS header, or wait for more data */ 2068 if (rxm->offset + prot->prepend_size > skb->len) 2069 return 0; 2070 2071 /* Sanity-check size of on-stack buffer. */ 2072 if (WARN_ON(prot->prepend_size > sizeof(header))) { 2073 ret = -EINVAL; 2074 goto read_failure; 2075 } 2076 2077 /* Linearize header to local buffer */ 2078 ret = skb_copy_bits(skb, rxm->offset, header, prot->prepend_size); 2079 2080 if (ret < 0) 2081 goto read_failure; 2082 2083 ctx->control = header[0]; 2084 2085 data_len = ((header[4] & 0xFF) | (header[3] << 8)); 2086 2087 cipher_overhead = prot->tag_size; 2088 if (prot->version != TLS_1_3_VERSION) 2089 cipher_overhead += prot->iv_size; 2090 2091 if (data_len > TLS_MAX_PAYLOAD_SIZE + cipher_overhead + 2092 prot->tail_size) { 2093 ret = -EMSGSIZE; 2094 goto read_failure; 2095 } 2096 if (data_len < cipher_overhead) { 2097 ret = -EBADMSG; 2098 goto read_failure; 2099 } 2100 2101 /* Note that both TLS1.3 and TLS1.2 use TLS_1_2 version here */ 2102 if (header[1] != TLS_1_2_VERSION_MINOR || 2103 header[2] != TLS_1_2_VERSION_MAJOR) { 2104 ret = -EINVAL; 2105 goto read_failure; 2106 } 2107 2108 tls_device_rx_resync_new_rec(strp->sk, data_len + TLS_HEADER_SIZE, 2109 TCP_SKB_CB(skb)->seq + rxm->offset); 2110 return data_len + TLS_HEADER_SIZE; 2111 2112read_failure: 2113 tls_err_abort(strp->sk, ret); 2114 2115 return ret; 2116} 2117 2118static void tls_queue(struct strparser *strp, struct sk_buff *skb) 2119{ 2120 struct tls_context *tls_ctx = tls_get_ctx(strp->sk); 2121 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 2122 2123 ctx->decrypted = 0; 2124 2125 ctx->recv_pkt = skb; 2126 strp_pause(strp); 2127 2128 ctx->saved_data_ready(strp->sk); 2129} 2130 2131static void tls_data_ready(struct sock *sk) 2132{ 2133 struct tls_context *tls_ctx = tls_get_ctx(sk); 2134 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 2135 struct sk_psock *psock; 2136 2137 strp_data_ready(&ctx->strp); 2138 2139 psock = sk_psock_get(sk); 2140 if (psock) { 2141 if (!list_empty(&psock->ingress_msg)) 2142 ctx->saved_data_ready(sk); 2143 sk_psock_put(sk, psock); 2144 } 2145} 2146 2147void tls_sw_cancel_work_tx(struct tls_context *tls_ctx) 2148{ 2149 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 2150 2151 set_bit(BIT_TX_CLOSING, &ctx->tx_bitmask); 2152 set_bit(BIT_TX_SCHEDULED, &ctx->tx_bitmask); 2153 cancel_delayed_work_sync(&ctx->tx_work.work); 2154} 2155 2156void tls_sw_release_resources_tx(struct sock *sk) 2157{ 2158 struct tls_context *tls_ctx = tls_get_ctx(sk); 2159 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 2160 struct tls_rec *rec, *tmp; 2161 int pending; 2162 2163 /* Wait for any pending async encryptions to complete */ 2164 spin_lock_bh(&ctx->encrypt_compl_lock); 2165 ctx->async_notify = true; 2166 pending = atomic_read(&ctx->encrypt_pending); 2167 spin_unlock_bh(&ctx->encrypt_compl_lock); 2168 2169 if (pending) 2170 crypto_wait_req(-EINPROGRESS, &ctx->async_wait); 2171 2172 tls_tx_records(sk, -1); 2173 2174 /* Free up un-sent records in tx_list. First, free 2175 * the partially sent record if any at head of tx_list. 2176 */ 2177 if (tls_ctx->partially_sent_record) { 2178 tls_free_partial_record(sk, tls_ctx); 2179 rec = list_first_entry(&ctx->tx_list, 2180 struct tls_rec, list); 2181 list_del(&rec->list); 2182 sk_msg_free(sk, &rec->msg_plaintext); 2183 kfree(rec); 2184 } 2185 2186 list_for_each_entry_safe(rec, tmp, &ctx->tx_list, list) { 2187 list_del(&rec->list); 2188 sk_msg_free(sk, &rec->msg_encrypted); 2189 sk_msg_free(sk, &rec->msg_plaintext); 2190 kfree(rec); 2191 } 2192 2193 crypto_free_aead(ctx->aead_send); 2194 tls_free_open_rec(sk); 2195} 2196 2197void tls_sw_free_ctx_tx(struct tls_context *tls_ctx) 2198{ 2199 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 2200 2201 kfree(ctx); 2202} 2203 2204void tls_sw_release_resources_rx(struct sock *sk) 2205{ 2206 struct tls_context *tls_ctx = tls_get_ctx(sk); 2207 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 2208 2209 kfree(tls_ctx->rx.rec_seq); 2210 kfree(tls_ctx->rx.iv); 2211 2212 if (ctx->aead_recv) { 2213 kfree_skb(ctx->recv_pkt); 2214 ctx->recv_pkt = NULL; 2215 skb_queue_purge(&ctx->rx_list); 2216 crypto_free_aead(ctx->aead_recv); 2217 strp_stop(&ctx->strp); 2218 /* If tls_sw_strparser_arm() was not called (cleanup paths) 2219 * we still want to strp_stop(), but sk->sk_data_ready was 2220 * never swapped. 2221 */ 2222 if (ctx->saved_data_ready) { 2223 write_lock_bh(&sk->sk_callback_lock); 2224 sk->sk_data_ready = ctx->saved_data_ready; 2225 write_unlock_bh(&sk->sk_callback_lock); 2226 } 2227 } 2228} 2229 2230void tls_sw_strparser_done(struct tls_context *tls_ctx) 2231{ 2232 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 2233 2234 strp_done(&ctx->strp); 2235} 2236 2237void tls_sw_free_ctx_rx(struct tls_context *tls_ctx) 2238{ 2239 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 2240 2241 kfree(ctx); 2242} 2243 2244void tls_sw_free_resources_rx(struct sock *sk) 2245{ 2246 struct tls_context *tls_ctx = tls_get_ctx(sk); 2247 2248 tls_sw_release_resources_rx(sk); 2249 tls_sw_free_ctx_rx(tls_ctx); 2250} 2251 2252/* The work handler to transmitt the encrypted records in tx_list */ 2253static void tx_work_handler(struct work_struct *work) 2254{ 2255 struct delayed_work *delayed_work = to_delayed_work(work); 2256 struct tx_work *tx_work = container_of(delayed_work, 2257 struct tx_work, work); 2258 struct sock *sk = tx_work->sk; 2259 struct tls_context *tls_ctx = tls_get_ctx(sk); 2260 struct tls_sw_context_tx *ctx; 2261 2262 if (unlikely(!tls_ctx)) 2263 return; 2264 2265 ctx = tls_sw_ctx_tx(tls_ctx); 2266 if (test_bit(BIT_TX_CLOSING, &ctx->tx_bitmask)) 2267 return; 2268 2269 if (!test_and_clear_bit(BIT_TX_SCHEDULED, &ctx->tx_bitmask)) 2270 return; 2271 2272 if (mutex_trylock(&tls_ctx->tx_lock)) { 2273 lock_sock(sk); 2274 tls_tx_records(sk, -1); 2275 release_sock(sk); 2276 mutex_unlock(&tls_ctx->tx_lock); 2277 } else if (!test_and_set_bit(BIT_TX_SCHEDULED, &ctx->tx_bitmask)) { 2278 /* Someone is holding the tx_lock, they will likely run Tx 2279 * and cancel the work on their way out of the lock section. 2280 * Schedule a long delay just in case. 2281 */ 2282 schedule_delayed_work(&ctx->tx_work.work, msecs_to_jiffies(10)); 2283 } 2284} 2285 2286void tls_sw_write_space(struct sock *sk, struct tls_context *ctx) 2287{ 2288 struct tls_sw_context_tx *tx_ctx = tls_sw_ctx_tx(ctx); 2289 2290 /* Schedule the transmission if tx list is ready */ 2291 if (is_tx_ready(tx_ctx) && 2292 !test_and_set_bit(BIT_TX_SCHEDULED, &tx_ctx->tx_bitmask)) 2293 schedule_delayed_work(&tx_ctx->tx_work.work, 0); 2294} 2295 2296void tls_sw_strparser_arm(struct sock *sk, struct tls_context *tls_ctx) 2297{ 2298 struct tls_sw_context_rx *rx_ctx = tls_sw_ctx_rx(tls_ctx); 2299 2300 write_lock_bh(&sk->sk_callback_lock); 2301 rx_ctx->saved_data_ready = sk->sk_data_ready; 2302 sk->sk_data_ready = tls_data_ready; 2303 write_unlock_bh(&sk->sk_callback_lock); 2304 2305 strp_check_rcv(&rx_ctx->strp); 2306} 2307 2308int tls_set_sw_offload(struct sock *sk, struct tls_context *ctx, int tx) 2309{ 2310 struct tls_context *tls_ctx = tls_get_ctx(sk); 2311 struct tls_prot_info *prot = &tls_ctx->prot_info; 2312 struct tls_crypto_info *crypto_info; 2313 struct tls12_crypto_info_aes_gcm_128 *gcm_128_info; 2314 struct tls12_crypto_info_aes_gcm_256 *gcm_256_info; 2315 struct tls12_crypto_info_aes_ccm_128 *ccm_128_info; 2316 struct tls_sw_context_tx *sw_ctx_tx = NULL; 2317 struct tls_sw_context_rx *sw_ctx_rx = NULL; 2318 struct cipher_context *cctx; 2319 struct crypto_aead **aead; 2320 struct strp_callbacks cb; 2321 u16 nonce_size, tag_size, iv_size, rec_seq_size, salt_size; 2322 struct crypto_tfm *tfm; 2323 char *iv, *rec_seq, *key, *salt, *cipher_name; 2324 size_t keysize; 2325 int rc = 0; 2326 2327 if (!ctx) { 2328 rc = -EINVAL; 2329 goto out; 2330 } 2331 2332 if (tx) { 2333 if (!ctx->priv_ctx_tx) { 2334 sw_ctx_tx = kzalloc(sizeof(*sw_ctx_tx), GFP_KERNEL); 2335 if (!sw_ctx_tx) { 2336 rc = -ENOMEM; 2337 goto out; 2338 } 2339 ctx->priv_ctx_tx = sw_ctx_tx; 2340 } else { 2341 sw_ctx_tx = 2342 (struct tls_sw_context_tx *)ctx->priv_ctx_tx; 2343 } 2344 } else { 2345 if (!ctx->priv_ctx_rx) { 2346 sw_ctx_rx = kzalloc(sizeof(*sw_ctx_rx), GFP_KERNEL); 2347 if (!sw_ctx_rx) { 2348 rc = -ENOMEM; 2349 goto out; 2350 } 2351 ctx->priv_ctx_rx = sw_ctx_rx; 2352 } else { 2353 sw_ctx_rx = 2354 (struct tls_sw_context_rx *)ctx->priv_ctx_rx; 2355 } 2356 } 2357 2358 if (tx) { 2359 crypto_init_wait(&sw_ctx_tx->async_wait); 2360 spin_lock_init(&sw_ctx_tx->encrypt_compl_lock); 2361 crypto_info = &ctx->crypto_send.info; 2362 cctx = &ctx->tx; 2363 aead = &sw_ctx_tx->aead_send; 2364 INIT_LIST_HEAD(&sw_ctx_tx->tx_list); 2365 INIT_DELAYED_WORK(&sw_ctx_tx->tx_work.work, tx_work_handler); 2366 sw_ctx_tx->tx_work.sk = sk; 2367 } else { 2368 crypto_init_wait(&sw_ctx_rx->async_wait); 2369 spin_lock_init(&sw_ctx_rx->decrypt_compl_lock); 2370 crypto_info = &ctx->crypto_recv.info; 2371 cctx = &ctx->rx; 2372 skb_queue_head_init(&sw_ctx_rx->rx_list); 2373 aead = &sw_ctx_rx->aead_recv; 2374 } 2375 2376 switch (crypto_info->cipher_type) { 2377 case TLS_CIPHER_AES_GCM_128: { 2378 nonce_size = TLS_CIPHER_AES_GCM_128_IV_SIZE; 2379 tag_size = TLS_CIPHER_AES_GCM_128_TAG_SIZE; 2380 iv_size = TLS_CIPHER_AES_GCM_128_IV_SIZE; 2381 iv = ((struct tls12_crypto_info_aes_gcm_128 *)crypto_info)->iv; 2382 rec_seq_size = TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE; 2383 rec_seq = 2384 ((struct tls12_crypto_info_aes_gcm_128 *)crypto_info)->rec_seq; 2385 gcm_128_info = 2386 (struct tls12_crypto_info_aes_gcm_128 *)crypto_info; 2387 keysize = TLS_CIPHER_AES_GCM_128_KEY_SIZE; 2388 key = gcm_128_info->key; 2389 salt = gcm_128_info->salt; 2390 salt_size = TLS_CIPHER_AES_GCM_128_SALT_SIZE; 2391 cipher_name = "gcm(aes)"; 2392 break; 2393 } 2394 case TLS_CIPHER_AES_GCM_256: { 2395 nonce_size = TLS_CIPHER_AES_GCM_256_IV_SIZE; 2396 tag_size = TLS_CIPHER_AES_GCM_256_TAG_SIZE; 2397 iv_size = TLS_CIPHER_AES_GCM_256_IV_SIZE; 2398 iv = ((struct tls12_crypto_info_aes_gcm_256 *)crypto_info)->iv; 2399 rec_seq_size = TLS_CIPHER_AES_GCM_256_REC_SEQ_SIZE; 2400 rec_seq = 2401 ((struct tls12_crypto_info_aes_gcm_256 *)crypto_info)->rec_seq; 2402 gcm_256_info = 2403 (struct tls12_crypto_info_aes_gcm_256 *)crypto_info; 2404 keysize = TLS_CIPHER_AES_GCM_256_KEY_SIZE; 2405 key = gcm_256_info->key; 2406 salt = gcm_256_info->salt; 2407 salt_size = TLS_CIPHER_AES_GCM_256_SALT_SIZE; 2408 cipher_name = "gcm(aes)"; 2409 break; 2410 } 2411 case TLS_CIPHER_AES_CCM_128: { 2412 nonce_size = TLS_CIPHER_AES_CCM_128_IV_SIZE; 2413 tag_size = TLS_CIPHER_AES_CCM_128_TAG_SIZE; 2414 iv_size = TLS_CIPHER_AES_CCM_128_IV_SIZE; 2415 iv = ((struct tls12_crypto_info_aes_ccm_128 *)crypto_info)->iv; 2416 rec_seq_size = TLS_CIPHER_AES_CCM_128_REC_SEQ_SIZE; 2417 rec_seq = 2418 ((struct tls12_crypto_info_aes_ccm_128 *)crypto_info)->rec_seq; 2419 ccm_128_info = 2420 (struct tls12_crypto_info_aes_ccm_128 *)crypto_info; 2421 keysize = TLS_CIPHER_AES_CCM_128_KEY_SIZE; 2422 key = ccm_128_info->key; 2423 salt = ccm_128_info->salt; 2424 salt_size = TLS_CIPHER_AES_CCM_128_SALT_SIZE; 2425 cipher_name = "ccm(aes)"; 2426 break; 2427 } 2428 default: 2429 rc = -EINVAL; 2430 goto free_priv; 2431 } 2432 2433 /* Sanity-check the sizes for stack allocations. */ 2434 if (iv_size > MAX_IV_SIZE || nonce_size > MAX_IV_SIZE || 2435 rec_seq_size > TLS_MAX_REC_SEQ_SIZE) { 2436 rc = -EINVAL; 2437 goto free_priv; 2438 } 2439 2440 if (crypto_info->version == TLS_1_3_VERSION) { 2441 nonce_size = 0; 2442 prot->aad_size = TLS_HEADER_SIZE; 2443 prot->tail_size = 1; 2444 } else { 2445 prot->aad_size = TLS_AAD_SPACE_SIZE; 2446 prot->tail_size = 0; 2447 } 2448 2449 prot->version = crypto_info->version; 2450 prot->cipher_type = crypto_info->cipher_type; 2451 prot->prepend_size = TLS_HEADER_SIZE + nonce_size; 2452 prot->tag_size = tag_size; 2453 prot->overhead_size = prot->prepend_size + 2454 prot->tag_size + prot->tail_size; 2455 prot->iv_size = iv_size; 2456 prot->salt_size = salt_size; 2457 cctx->iv = kmalloc(iv_size + salt_size, GFP_KERNEL); 2458 if (!cctx->iv) { 2459 rc = -ENOMEM; 2460 goto free_priv; 2461 } 2462 /* Note: 128 & 256 bit salt are the same size */ 2463 prot->rec_seq_size = rec_seq_size; 2464 memcpy(cctx->iv, salt, salt_size); 2465 memcpy(cctx->iv + salt_size, iv, iv_size); 2466 cctx->rec_seq = kmemdup(rec_seq, rec_seq_size, GFP_KERNEL); 2467 if (!cctx->rec_seq) { 2468 rc = -ENOMEM; 2469 goto free_iv; 2470 } 2471 2472 if (!*aead) { 2473 *aead = crypto_alloc_aead(cipher_name, 0, 0); 2474 if (IS_ERR(*aead)) { 2475 rc = PTR_ERR(*aead); 2476 *aead = NULL; 2477 goto free_rec_seq; 2478 } 2479 } 2480 2481 ctx->push_pending_record = tls_sw_push_pending_record; 2482 2483 rc = crypto_aead_setkey(*aead, key, keysize); 2484 2485 if (rc) 2486 goto free_aead; 2487 2488 rc = crypto_aead_setauthsize(*aead, prot->tag_size); 2489 if (rc) 2490 goto free_aead; 2491 2492 if (sw_ctx_rx) { 2493 tfm = crypto_aead_tfm(sw_ctx_rx->aead_recv); 2494 2495 if (crypto_info->version == TLS_1_3_VERSION) 2496 sw_ctx_rx->async_capable = 0; 2497 else 2498 sw_ctx_rx->async_capable = 2499 !!(tfm->__crt_alg->cra_flags & 2500 CRYPTO_ALG_ASYNC); 2501 2502 /* Set up strparser */ 2503 memset(&cb, 0, sizeof(cb)); 2504 cb.rcv_msg = tls_queue; 2505 cb.parse_msg = tls_read_size; 2506 2507 strp_init(&sw_ctx_rx->strp, sk, &cb); 2508 } 2509 2510 goto out; 2511 2512free_aead: 2513 crypto_free_aead(*aead); 2514 *aead = NULL; 2515free_rec_seq: 2516 kfree(cctx->rec_seq); 2517 cctx->rec_seq = NULL; 2518free_iv: 2519 kfree(cctx->iv); 2520 cctx->iv = NULL; 2521free_priv: 2522 if (tx) { 2523 kfree(ctx->priv_ctx_tx); 2524 ctx->priv_ctx_tx = NULL; 2525 } else { 2526 kfree(ctx->priv_ctx_rx); 2527 ctx->priv_ctx_rx = NULL; 2528 } 2529out: 2530 return rc; 2531} 2532