1/*
2 * nghttp2 - HTTP/2 C Library
3 *
4 * Copyright (c) 2021 Tatsuhiro Tsujikawa
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining
7 * a copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sublicense, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice shall be
15 * included in all copies or substantial portions of the Software.
16 *
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
20 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
24 */
25#include "shrpx_quic_connection_handler.h"
26
27#include <openssl/rand.h>
28
29#include <ngtcp2/ngtcp2.h>
30#include <ngtcp2/ngtcp2_crypto.h>
31
32#include "shrpx_worker.h"
33#include "shrpx_client_handler.h"
34#include "shrpx_log.h"
35#include "shrpx_http3_upstream.h"
36#include "shrpx_connection_handler.h"
37
38namespace shrpx {
39
40namespace {
41void stateless_reset_bucket_regen_timercb(struct ev_loop *loop, ev_timer *w,
42                                          int revents) {
43  auto quic_conn_handler = static_cast<QUICConnectionHandler *>(w->data);
44
45  quic_conn_handler->on_stateless_reset_bucket_regen();
46}
47} // namespace
48
49QUICConnectionHandler::QUICConnectionHandler(Worker *worker)
50    : worker_{worker},
51      stateless_reset_bucket_{SHRPX_QUIC_STATELESS_RESET_BURST} {
52  ev_timer_init(&stateless_reset_bucket_regen_timer_,
53                stateless_reset_bucket_regen_timercb, 0., 1.);
54  stateless_reset_bucket_regen_timer_.data = this;
55}
56
57QUICConnectionHandler::~QUICConnectionHandler() {
58  ev_timer_stop(worker_->get_loop(), &stateless_reset_bucket_regen_timer_);
59}
60
61int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr,
62                                         const Address &remote_addr,
63                                         const Address &local_addr,
64                                         const ngtcp2_pkt_info &pi,
65                                         const uint8_t *data, size_t datalen) {
66  int rv;
67  ngtcp2_version_cid vc;
68
69  rv = ngtcp2_pkt_decode_version_cid(&vc, data, datalen, SHRPX_QUIC_SCIDLEN);
70  switch (rv) {
71  case 0:
72    break;
73  case NGTCP2_ERR_VERSION_NEGOTIATION:
74    send_version_negotiation(faddr, vc.version, vc.dcid, vc.dcidlen, vc.scid,
75                             vc.scidlen, remote_addr, local_addr);
76
77    return 0;
78  default:
79    return 0;
80  }
81
82  auto config = get_config();
83
84  ngtcp2_cid dcid_key;
85  ngtcp2_cid_init(&dcid_key, vc.dcid, vc.dcidlen);
86
87  auto conn_handler = worker_->get_connection_handler();
88
89  ClientHandler *handler;
90
91  auto &quicconf = config->quic;
92
93  auto it = connections_.find(dcid_key);
94  if (it == std::end(connections_)) {
95    auto cwit = close_waits_.find(dcid_key);
96    if (cwit != std::end(close_waits_)) {
97      auto cw = (*cwit).second;
98
99      cw->handle_packet(faddr, remote_addr, local_addr, pi, data, datalen);
100
101      return 0;
102    }
103
104    if (data[0] & 0x80) {
105      if (generate_quic_hashed_connection_id(dcid_key, remote_addr, local_addr,
106                                             dcid_key) != 0) {
107        return 0;
108      }
109
110      it = connections_.find(dcid_key);
111      if (it == std::end(connections_)) {
112        auto cwit = close_waits_.find(dcid_key);
113        if (cwit != std::end(close_waits_)) {
114          auto cw = (*cwit).second;
115
116          cw->handle_packet(faddr, remote_addr, local_addr, pi, data, datalen);
117
118          return 0;
119        }
120      }
121    }
122  }
123
124  if (it == std::end(connections_)) {
125    std::array<uint8_t, SHRPX_QUIC_DECRYPTED_DCIDLEN> decrypted_dcid;
126
127    auto &qkms = conn_handler->get_quic_keying_materials();
128    const QUICKeyingMaterial *qkm = nullptr;
129
130    if (vc.dcidlen == SHRPX_QUIC_SCIDLEN) {
131      qkm = select_quic_keying_material(
132          *qkms.get(), vc.dcid[0] & SHRPX_QUIC_DCID_KM_ID_MASK);
133
134      if (decrypt_quic_connection_id(decrypted_dcid.data(),
135                                     vc.dcid + SHRPX_QUIC_CID_PREFIX_OFFSET,
136                                     qkm->cid_encryption_key.data()) != 0) {
137        return 0;
138      }
139
140      if (qkm != &qkms->keying_materials.front() ||
141          !std::equal(std::begin(decrypted_dcid),
142                      std::begin(decrypted_dcid) + SHRPX_QUIC_CID_PREFIXLEN,
143                      worker_->get_cid_prefix())) {
144        auto quic_lwp =
145            conn_handler->match_quic_lingering_worker_process_cid_prefix(
146                decrypted_dcid.data(), decrypted_dcid.size());
147        if (quic_lwp) {
148          if (conn_handler->forward_quic_packet_to_lingering_worker_process(
149                  quic_lwp, remote_addr, local_addr, pi, data, datalen) == 0) {
150            return 0;
151          }
152
153          return 0;
154        }
155      }
156    }
157
158    // new connection
159
160    auto &upstreamconf = config->conn.upstream;
161    if (worker_->get_worker_stat()->num_connections >=
162        upstreamconf.worker_connections) {
163      if (LOG_ENABLED(INFO)) {
164        LOG(INFO) << "Too many connections >="
165                  << upstreamconf.worker_connections;
166      }
167
168      return 0;
169    }
170
171    ngtcp2_pkt_hd hd;
172    ngtcp2_cid odcid, *podcid = nullptr;
173    const uint8_t *token = nullptr;
174    size_t tokenlen = 0;
175    ngtcp2_token_type token_type = NGTCP2_TOKEN_TYPE_UNKNOWN;
176
177    switch (ngtcp2_accept(&hd, data, datalen)) {
178    case 0: {
179      // If we get Initial and it has the CID prefix of this worker,
180      // it is likely that client is intentionally use the prefix.
181      // Just drop it.
182      if (vc.dcidlen == SHRPX_QUIC_SCIDLEN) {
183        if (qkm != &qkms->keying_materials.front()) {
184          qkm = &qkms->keying_materials.front();
185
186          if (decrypt_quic_connection_id(decrypted_dcid.data(),
187                                         vc.dcid + SHRPX_QUIC_CID_PREFIX_OFFSET,
188                                         qkm->cid_encryption_key.data()) != 0) {
189            return 0;
190          }
191        }
192
193        if (std::equal(std::begin(decrypted_dcid),
194                       std::begin(decrypted_dcid) + SHRPX_QUIC_CID_PREFIXLEN,
195                       worker_->get_cid_prefix())) {
196          return 0;
197        }
198      }
199
200      if (worker_->get_graceful_shutdown()) {
201        send_connection_close(faddr, hd.version, hd.dcid, hd.scid, remote_addr,
202                              local_addr, NGTCP2_CONNECTION_REFUSED,
203                              datalen * 3);
204        return 0;
205      }
206
207      if (hd.tokenlen == 0) {
208        if (quicconf.upstream.require_token) {
209          send_retry(faddr, vc.version, vc.dcid, vc.dcidlen, vc.scid,
210                     vc.scidlen, remote_addr, local_addr, datalen * 3);
211
212          return 0;
213        }
214
215        break;
216      }
217
218      switch (hd.token[0]) {
219      case NGTCP2_CRYPTO_TOKEN_MAGIC_RETRY: {
220        if (vc.dcidlen != SHRPX_QUIC_SCIDLEN) {
221          // Initial packets with Retry token must have DCID chosen by
222          // server.
223          return 0;
224        }
225
226        auto qkm = select_quic_keying_material(
227            *qkms.get(), vc.dcid[0] & SHRPX_QUIC_DCID_KM_ID_MASK);
228
229        if (verify_retry_token(odcid, hd.token, hd.tokenlen, hd.version,
230                               hd.dcid, &remote_addr.su.sa, remote_addr.len,
231                               qkm->secret.data(), qkm->secret.size()) != 0) {
232          if (LOG_ENABLED(INFO)) {
233            LOG(INFO) << "Failed to validate Retry token from remote="
234                      << util::to_numeric_addr(&remote_addr);
235          }
236
237          // 2nd Retry packet is not allowed, so send CONNECTION_CLOSE
238          // with INVALID_TOKEN.
239          send_connection_close(faddr, hd.version, hd.dcid, hd.scid,
240                                remote_addr, local_addr, NGTCP2_INVALID_TOKEN,
241                                datalen * 3);
242          return 0;
243        }
244
245        if (LOG_ENABLED(INFO)) {
246          LOG(INFO) << "Successfully validated Retry token from remote="
247                    << util::to_numeric_addr(&remote_addr);
248        }
249
250        podcid = &odcid;
251        token = hd.token;
252        tokenlen = hd.tokenlen;
253        token_type = NGTCP2_TOKEN_TYPE_RETRY;
254
255        break;
256      }
257      case NGTCP2_CRYPTO_TOKEN_MAGIC_REGULAR: {
258        // If a token is a regular token, it must be at least
259        // NGTCP2_MIN_INITIAL_DCIDLEN bytes long.
260        if (vc.dcidlen < NGTCP2_MIN_INITIAL_DCIDLEN) {
261          return 0;
262        }
263
264        if (hd.tokenlen != NGTCP2_CRYPTO_MAX_REGULAR_TOKENLEN + 1) {
265          if (LOG_ENABLED(INFO)) {
266            LOG(INFO) << "Failed to validate token from remote="
267                      << util::to_numeric_addr(&remote_addr);
268          }
269
270          if (quicconf.upstream.require_token) {
271            send_retry(faddr, vc.version, vc.dcid, vc.dcidlen, vc.scid,
272                       vc.scidlen, remote_addr, local_addr, datalen * 3);
273
274            return 0;
275          }
276
277          break;
278        }
279
280        auto qkm = select_quic_keying_material(
281            *qkms.get(), hd.token[NGTCP2_CRYPTO_MAX_REGULAR_TOKENLEN]);
282
283        if (verify_token(hd.token, hd.tokenlen - 1, &remote_addr.su.sa,
284                         remote_addr.len, qkm->secret.data(),
285                         qkm->secret.size()) != 0) {
286          if (LOG_ENABLED(INFO)) {
287            LOG(INFO) << "Failed to validate token from remote="
288                      << util::to_numeric_addr(&remote_addr);
289          }
290
291          if (quicconf.upstream.require_token) {
292            send_retry(faddr, vc.version, vc.dcid, vc.dcidlen, vc.scid,
293                       vc.scidlen, remote_addr, local_addr, datalen * 3);
294
295            return 0;
296          }
297
298          break;
299        }
300
301        if (LOG_ENABLED(INFO)) {
302          LOG(INFO) << "Successfully validated token from remote="
303                    << util::to_numeric_addr(&remote_addr);
304        }
305
306        token = hd.token;
307        tokenlen = hd.tokenlen;
308        token_type = NGTCP2_TOKEN_TYPE_NEW_TOKEN;
309
310        break;
311      }
312      default:
313        if (quicconf.upstream.require_token) {
314          send_retry(faddr, vc.version, vc.dcid, vc.dcidlen, vc.scid,
315                     vc.scidlen, remote_addr, local_addr, datalen * 3);
316
317          return 0;
318        }
319
320        break;
321      }
322
323      break;
324    }
325    default:
326      if (!config->single_thread && !(data[0] & 0x80) &&
327          vc.dcidlen == SHRPX_QUIC_SCIDLEN &&
328          !std::equal(std::begin(decrypted_dcid),
329                      std::begin(decrypted_dcid) + SHRPX_QUIC_CID_PREFIXLEN,
330                      worker_->get_cid_prefix())) {
331        if (conn_handler->forward_quic_packet(faddr, remote_addr, local_addr,
332                                              pi, decrypted_dcid.data(), data,
333                                              datalen) == 0) {
334          return 0;
335        }
336      }
337
338      if (!(data[0] & 0x80)) {
339        // TODO Must be rate limited
340        send_stateless_reset(faddr, vc.dcid, vc.dcidlen, remote_addr,
341                             local_addr);
342      }
343
344      return 0;
345    }
346
347    handler = handle_new_connection(faddr, remote_addr, local_addr, hd, podcid,
348                                    token, tokenlen, token_type);
349    if (handler == nullptr) {
350      return 0;
351    }
352  } else {
353    handler = (*it).second;
354  }
355
356  if (handler->read_quic(faddr, remote_addr, local_addr, pi, data, datalen) !=
357      0) {
358    delete handler;
359    return 0;
360  }
361
362  handler->signal_write();
363
364  return 0;
365}
366
367ClientHandler *QUICConnectionHandler::handle_new_connection(
368    const UpstreamAddr *faddr, const Address &remote_addr,
369    const Address &local_addr, const ngtcp2_pkt_hd &hd, const ngtcp2_cid *odcid,
370    const uint8_t *token, size_t tokenlen, ngtcp2_token_type token_type) {
371  std::array<char, NI_MAXHOST> host;
372  std::array<char, NI_MAXSERV> service;
373  int rv;
374
375  rv = getnameinfo(&remote_addr.su.sa, remote_addr.len, host.data(),
376                   host.size(), service.data(), service.size(),
377                   NI_NUMERICHOST | NI_NUMERICSERV);
378  if (rv != 0) {
379    LOG(ERROR) << "getnameinfo() failed: " << gai_strerror(rv);
380
381    return nullptr;
382  }
383
384  auto ssl_ctx = worker_->get_quic_sv_ssl_ctx();
385
386  assert(ssl_ctx);
387
388  auto ssl = tls::create_ssl(ssl_ctx);
389  if (ssl == nullptr) {
390    return nullptr;
391  }
392
393#if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)
394  assert(SSL_is_quic(ssl));
395#endif // OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)
396
397  SSL_set_accept_state(ssl);
398
399  auto config = get_config();
400  auto &quicconf = config->quic;
401
402  if (quicconf.upstream.early_data) {
403#if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)
404    SSL_set_quic_early_data_enabled(ssl, 1);
405#else  // !(OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL))
406    SSL_set_early_data_enabled(ssl, 1);
407#endif // !(OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL))
408  }
409
410  // Disable TLS session ticket if we don't have working ticket
411  // keys.
412  if (!worker_->get_ticket_keys()) {
413    SSL_set_options(ssl, SSL_OP_NO_TICKET);
414  }
415
416  auto handler = std::make_unique<ClientHandler>(
417      worker_, faddr->fd, ssl, StringRef{host.data()},
418      StringRef{service.data()}, remote_addr.su.sa.sa_family, faddr);
419
420  auto upstream = std::make_unique<Http3Upstream>(handler.get());
421  if (upstream->init(faddr, remote_addr, local_addr, hd, odcid, token, tokenlen,
422                     token_type) != 0) {
423    return nullptr;
424  }
425
426  handler->setup_http3_upstream(std::move(upstream));
427
428  return handler.release();
429}
430
431namespace {
432uint32_t generate_reserved_version(const Address &addr, uint32_t version) {
433  uint32_t h = 0x811C9DC5u;
434  const uint8_t *p = reinterpret_cast<const uint8_t *>(&addr.su.sa);
435  const uint8_t *ep = p + addr.len;
436
437  for (; p != ep; ++p) {
438    h ^= *p;
439    h *= 0x01000193u;
440  }
441
442  version = htonl(version);
443  p = (const uint8_t *)&version;
444  ep = p + sizeof(version);
445
446  for (; p != ep; ++p) {
447    h ^= *p;
448    h *= 0x01000193u;
449  }
450
451  h &= 0xf0f0f0f0u;
452  h |= 0x0a0a0a0au;
453
454  return h;
455}
456} // namespace
457
458int QUICConnectionHandler::send_retry(
459    const UpstreamAddr *faddr, uint32_t version, const uint8_t *ini_dcid,
460    size_t ini_dcidlen, const uint8_t *ini_scid, size_t ini_scidlen,
461    const Address &remote_addr, const Address &local_addr, size_t max_pktlen) {
462  std::array<char, NI_MAXHOST> host;
463  std::array<char, NI_MAXSERV> port;
464
465  if (getnameinfo(&remote_addr.su.sa, remote_addr.len, host.data(), host.size(),
466                  port.data(), port.size(),
467                  NI_NUMERICHOST | NI_NUMERICSERV) != 0) {
468    return -1;
469  }
470
471  auto config = get_config();
472  auto &quicconf = config->quic;
473
474  auto conn_handler = worker_->get_connection_handler();
475  auto &qkms = conn_handler->get_quic_keying_materials();
476  auto &qkm = qkms->keying_materials.front();
477
478  ngtcp2_cid retry_scid;
479
480  if (generate_quic_retry_connection_id(retry_scid, SHRPX_QUIC_SCIDLEN,
481                                        quicconf.server_id.data(), qkm.id,
482                                        qkm.cid_encryption_key.data()) != 0) {
483    return -1;
484  }
485
486  std::array<uint8_t, NGTCP2_CRYPTO_MAX_RETRY_TOKENLEN> token;
487  size_t tokenlen;
488
489  ngtcp2_cid idcid, iscid;
490  ngtcp2_cid_init(&idcid, ini_dcid, ini_dcidlen);
491  ngtcp2_cid_init(&iscid, ini_scid, ini_scidlen);
492
493  if (generate_retry_token(token.data(), tokenlen, version, &remote_addr.su.sa,
494                           remote_addr.len, retry_scid, idcid,
495                           qkm.secret.data(), qkm.secret.size()) != 0) {
496    return -1;
497  }
498
499  std::vector<uint8_t> buf;
500  buf.resize(std::min(max_pktlen, static_cast<size_t>(256)));
501
502  auto nwrite =
503      ngtcp2_crypto_write_retry(buf.data(), buf.size(), version, &iscid,
504                                &retry_scid, &idcid, token.data(), tokenlen);
505  if (nwrite < 0) {
506    LOG(ERROR) << "ngtcp2_crypto_write_retry: " << ngtcp2_strerror(nwrite);
507    return -1;
508  }
509
510  buf.resize(nwrite);
511
512  quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len,
513                   &local_addr.su.sa, local_addr.len, ngtcp2_pkt_info{},
514                   buf.data(), buf.size(), 0);
515
516  if (generate_quic_hashed_connection_id(idcid, remote_addr, local_addr,
517                                         idcid) != 0) {
518    return -1;
519  }
520
521  auto d =
522      static_cast<ev_tstamp>(NGTCP2_DEFAULT_INITIAL_RTT * 3) / NGTCP2_SECONDS;
523
524  if (LOG_ENABLED(INFO)) {
525    LOG(INFO) << "Enter close-wait period " << d << "s with " << buf.size()
526              << " bytes sentinel packet";
527  }
528
529  auto cw = std::make_unique<CloseWait>(worker_, std::vector<ngtcp2_cid>{idcid},
530                                        std::move(buf), d);
531
532  add_close_wait(cw.release());
533
534  return 0;
535}
536
537int QUICConnectionHandler::send_version_negotiation(
538    const UpstreamAddr *faddr, uint32_t version, const uint8_t *ini_dcid,
539    size_t ini_dcidlen, const uint8_t *ini_scid, size_t ini_scidlen,
540    const Address &remote_addr, const Address &local_addr) {
541  std::array<uint32_t, 2> sv{
542      generate_reserved_version(remote_addr, version),
543      NGTCP2_PROTO_VER_V1,
544  };
545
546  std::array<uint8_t, NGTCP2_MAX_UDP_PAYLOAD_SIZE> buf;
547
548  uint8_t rand_byte;
549  util::random_bytes(&rand_byte, &rand_byte + 1, worker_->get_randgen());
550
551  auto nwrite = ngtcp2_pkt_write_version_negotiation(
552      buf.data(), buf.size(), rand_byte, ini_scid, ini_scidlen, ini_dcid,
553      ini_dcidlen, sv.data(), sv.size());
554  if (nwrite < 0) {
555    LOG(ERROR) << "ngtcp2_pkt_write_version_negotiation: "
556               << ngtcp2_strerror(nwrite);
557    return -1;
558  }
559
560  return quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len,
561                          &local_addr.su.sa, local_addr.len, ngtcp2_pkt_info{},
562                          buf.data(), nwrite, 0);
563}
564
565int QUICConnectionHandler::send_stateless_reset(const UpstreamAddr *faddr,
566                                                const uint8_t *dcid,
567                                                size_t dcidlen,
568                                                const Address &remote_addr,
569                                                const Address &local_addr) {
570  if (stateless_reset_bucket_ == 0) {
571    if (LOG_ENABLED(INFO)) {
572      LOG(INFO) << "Stateless Reset bucket has been depleted";
573    }
574
575    return 0;
576  }
577
578  --stateless_reset_bucket_;
579
580  if (!ev_is_active(&stateless_reset_bucket_regen_timer_)) {
581    ev_timer_again(worker_->get_loop(), &stateless_reset_bucket_regen_timer_);
582  }
583
584  int rv;
585  std::array<uint8_t, NGTCP2_STATELESS_RESET_TOKENLEN> token;
586  ngtcp2_cid cid;
587
588  ngtcp2_cid_init(&cid, dcid, dcidlen);
589
590  auto conn_handler = worker_->get_connection_handler();
591  auto &qkms = conn_handler->get_quic_keying_materials();
592  auto &qkm = qkms->keying_materials.front();
593
594  rv = generate_quic_stateless_reset_token(token.data(), cid, qkm.secret.data(),
595                                           qkm.secret.size());
596  if (rv != 0) {
597    return -1;
598  }
599
600  std::array<uint8_t, NGTCP2_MIN_STATELESS_RESET_RANDLEN> rand_bytes;
601
602  if (RAND_bytes(rand_bytes.data(), rand_bytes.size()) != 1) {
603    return -1;
604  }
605
606  std::array<uint8_t, NGTCP2_MAX_UDP_PAYLOAD_SIZE> buf;
607
608  auto nwrite =
609      ngtcp2_pkt_write_stateless_reset(buf.data(), buf.size(), token.data(),
610                                       rand_bytes.data(), rand_bytes.size());
611  if (nwrite < 0) {
612    LOG(ERROR) << "ngtcp2_pkt_write_stateless_reset: "
613               << ngtcp2_strerror(nwrite);
614    return -1;
615  }
616
617  if (LOG_ENABLED(INFO)) {
618    LOG(INFO) << "Send stateless_reset to remote="
619              << util::to_numeric_addr(&remote_addr)
620              << " dcid=" << util::format_hex(dcid, dcidlen);
621  }
622
623  return quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len,
624                          &local_addr.su.sa, local_addr.len, ngtcp2_pkt_info{},
625                          buf.data(), nwrite, 0);
626}
627
628int QUICConnectionHandler::send_connection_close(
629    const UpstreamAddr *faddr, uint32_t version, const ngtcp2_cid &ini_dcid,
630    const ngtcp2_cid &ini_scid, const Address &remote_addr,
631    const Address &local_addr, uint64_t error_code, size_t max_pktlen) {
632  std::array<uint8_t, NGTCP2_MAX_UDP_PAYLOAD_SIZE> buf;
633
634  max_pktlen = std::min(max_pktlen, buf.size());
635
636  auto nwrite = ngtcp2_crypto_write_connection_close(
637      buf.data(), max_pktlen, version, &ini_scid, &ini_dcid, error_code,
638      nullptr, 0);
639  if (nwrite < 0) {
640    LOG(ERROR) << "ngtcp2_crypto_write_connection_close failed";
641    return -1;
642  }
643
644  if (LOG_ENABLED(INFO)) {
645    LOG(INFO) << "Send Initial CONNECTION_CLOSE with error_code=" << log::hex
646              << error_code << log::dec
647              << " to remote=" << util::to_numeric_addr(&remote_addr)
648              << " dcid=" << util::format_hex(ini_scid.data, ini_scid.datalen)
649              << " scid=" << util::format_hex(ini_dcid.data, ini_dcid.datalen);
650  }
651
652  return quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len,
653                          &local_addr.su.sa, local_addr.len, ngtcp2_pkt_info{},
654                          buf.data(), nwrite, 0);
655}
656
657void QUICConnectionHandler::add_connection_id(const ngtcp2_cid &cid,
658                                              ClientHandler *handler) {
659  connections_.emplace(cid, handler);
660}
661
662void QUICConnectionHandler::remove_connection_id(const ngtcp2_cid &cid) {
663  connections_.erase(cid);
664}
665
666void QUICConnectionHandler::add_close_wait(CloseWait *cw) {
667  for (auto &cid : cw->scids) {
668    close_waits_.emplace(cid, cw);
669  }
670}
671
672void QUICConnectionHandler::remove_close_wait(const CloseWait *cw) {
673  for (auto &cid : cw->scids) {
674    close_waits_.erase(cid);
675  }
676}
677
678void QUICConnectionHandler::on_stateless_reset_bucket_regen() {
679  assert(stateless_reset_bucket_ < SHRPX_QUIC_STATELESS_RESET_BURST);
680
681  if (++stateless_reset_bucket_ == SHRPX_QUIC_STATELESS_RESET_BURST) {
682    ev_timer_stop(worker_->get_loop(), &stateless_reset_bucket_regen_timer_);
683  }
684}
685
686static void close_wait_timeoutcb(struct ev_loop *loop, ev_timer *w,
687                                 int revents) {
688  auto cw = static_cast<CloseWait *>(w->data);
689
690  if (LOG_ENABLED(INFO)) {
691    LOG(INFO) << "close-wait period finished";
692  }
693
694  auto quic_conn_handler = cw->worker->get_quic_connection_handler();
695  quic_conn_handler->remove_close_wait(cw);
696
697  delete cw;
698}
699
700CloseWait::CloseWait(Worker *worker, std::vector<ngtcp2_cid> scids,
701                     std::vector<uint8_t> pkt, ev_tstamp period)
702    : worker{worker},
703      scids{std::move(scids)},
704      pkt{std::move(pkt)},
705      bytes_recv{0},
706      bytes_sent{0},
707      num_pkts_recv{0},
708      next_pkts_recv{1} {
709  ++worker->get_worker_stat()->num_close_waits;
710
711  ev_timer_init(&timer, close_wait_timeoutcb, period, 0.);
712  timer.data = this;
713
714  ev_timer_start(worker->get_loop(), &timer);
715}
716
717CloseWait::~CloseWait() {
718  auto loop = worker->get_loop();
719
720  ev_timer_stop(loop, &timer);
721
722  auto worker_stat = worker->get_worker_stat();
723  --worker_stat->num_close_waits;
724
725  if (worker->get_graceful_shutdown() && worker_stat->num_connections == 0 &&
726      worker_stat->num_close_waits == 0) {
727    ev_break(loop);
728  }
729}
730
731int CloseWait::handle_packet(const UpstreamAddr *faddr,
732                             const Address &remote_addr,
733                             const Address &local_addr,
734                             const ngtcp2_pkt_info &pi, const uint8_t *data,
735                             size_t datalen) {
736  if (pkt.empty()) {
737    return 0;
738  }
739
740  ++num_pkts_recv;
741  bytes_recv += datalen;
742
743  if (bytes_sent + pkt.size() > 3 * bytes_recv ||
744      next_pkts_recv > num_pkts_recv) {
745    return 0;
746  }
747
748  if (quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len,
749                       &local_addr.su.sa, local_addr.len, ngtcp2_pkt_info{},
750                       pkt.data(), pkt.size(), 0) != 0) {
751    return -1;
752  }
753
754  next_pkts_recv *= 2;
755  bytes_sent += pkt.size();
756
757  return 0;
758}
759
760} // namespace shrpx
761