1/*
2 * nghttp2 - HTTP/2 C Library
3 *
4 * Copyright (c) 2012 Tatsuhiro Tsujikawa
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining
7 * a copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sublicense, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice shall be
15 * included in all copies or substantial portions of the Software.
16 *
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
20 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
24 */
25#ifndef SHRPX_CONNECTION_HANDLER_H
26#define SHRPX_CONNECTION_HANDLER_H
27
28#include "shrpx.h"
29
30#include <sys/types.h>
31#ifdef HAVE_SYS_SOCKET_H
32#  include <sys/socket.h>
33#endif // HAVE_SYS_SOCKET_H
34
35#include <mutex>
36#include <memory>
37#include <vector>
38#include <random>
39#ifndef NOTHREADS
40#  include <future>
41#endif // NOTHREADS
42
43#ifdef HAVE_LIBBPF
44#  include <bpf/libbpf.h>
45#endif // HAVE_LIBBPF
46
47#include <openssl/ssl.h>
48
49#include <ev.h>
50
51#ifdef HAVE_NEVERBLEED
52#  include <neverbleed.h>
53#endif // HAVE_NEVERBLEED
54
55#include "shrpx_downstream_connection_pool.h"
56#include "shrpx_config.h"
57#include "shrpx_exec.h"
58
59namespace shrpx {
60
61class Http2Session;
62class ConnectBlocker;
63class AcceptHandler;
64class Worker;
65struct WorkerStat;
66struct TicketKeys;
67class MemcachedDispatcher;
68struct UpstreamAddr;
69
70namespace tls {
71
72class CertLookupTree;
73
74} // namespace tls
75
76struct OCSPUpdateContext {
77  // ocsp response buffer
78  std::vector<uint8_t> resp;
79  // Process running fetch-ocsp-response script
80  Process proc;
81  // index to ConnectionHandler::all_ssl_ctx_, which points to next
82  // SSL_CTX to update ocsp response cache.
83  size_t next;
84  ev_child chldev;
85  ev_io rev;
86  // errno encountered while processing response
87  int error;
88};
89
90// SerialEvent is an event sent from Worker thread.
91enum class SerialEventType {
92  NONE,
93  REPLACE_DOWNSTREAM,
94};
95
96struct SerialEvent {
97  // ctor for event uses DownstreamConfig
98  SerialEvent(SerialEventType type,
99              const std::shared_ptr<DownstreamConfig> &downstreamconf)
100      : type(type), downstreamconf(downstreamconf) {}
101
102  SerialEventType type;
103  std::shared_ptr<DownstreamConfig> downstreamconf;
104};
105
106#ifdef ENABLE_HTTP3
107#  ifdef HAVE_LIBBPF
108struct BPFRef {
109  bpf_object *obj;
110  bpf_map *reuseport_array;
111  bpf_map *cid_prefix_map;
112};
113#  endif // HAVE_LIBBPF
114
115// QUIC IPC message type.
116enum class QUICIPCType {
117  NONE,
118  // Send forwarded QUIC UDP datagram and its metadata.
119  DGRAM_FORWARD,
120};
121
122// WorkerProcesses which are in graceful shutdown period.
123struct QUICLingeringWorkerProcess {
124  QUICLingeringWorkerProcess(
125      std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>> cid_prefixes,
126      int quic_ipc_fd)
127      : cid_prefixes{std::move(cid_prefixes)}, quic_ipc_fd{quic_ipc_fd} {}
128
129  std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>> cid_prefixes;
130  // Socket to send QUIC IPC message to this worker process.
131  int quic_ipc_fd;
132};
133#endif // ENABLE_HTTP3
134
135class ConnectionHandler {
136public:
137  ConnectionHandler(struct ev_loop *loop, std::mt19937 &gen);
138  ~ConnectionHandler();
139  int handle_connection(int fd, sockaddr *addr, int addrlen,
140                        const UpstreamAddr *faddr);
141  // Creates Worker object for single threaded configuration.
142  int create_single_worker();
143  // Creates |num| Worker objects for multi threaded configuration.
144  // The |num| must be strictly more than 1.
145  int create_worker_thread(size_t num);
146  void
147  set_ticket_keys_to_worker(const std::shared_ptr<TicketKeys> &ticket_keys);
148  void worker_reopen_log_files();
149  void set_ticket_keys(std::shared_ptr<TicketKeys> ticket_keys);
150  const std::shared_ptr<TicketKeys> &get_ticket_keys() const;
151  struct ev_loop *get_loop() const;
152  Worker *get_single_worker() const;
153  void add_acceptor(std::unique_ptr<AcceptHandler> h);
154  void delete_acceptor();
155  void enable_acceptor();
156  void disable_acceptor();
157  void sleep_acceptor(ev_tstamp t);
158  void accept_pending_connection();
159  void graceful_shutdown_worker();
160  void set_graceful_shutdown(bool f);
161  bool get_graceful_shutdown() const;
162  void join_worker();
163
164  // Cancels ocsp update process
165  void cancel_ocsp_update();
166  // Starts ocsp update for certificate |cert_file|.
167  int start_ocsp_update(const char *cert_file);
168  // Reads incoming data from ocsp update process
169  void read_ocsp_chunk();
170  // Handles the completion of one ocsp update
171  void handle_ocsp_complete();
172  // Resets ocsp_;
173  void reset_ocsp();
174  // Proceeds to the next certificate's ocsp update.  If all
175  // certificates' ocsp update has been done, schedule next ocsp
176  // update.
177  void proceed_next_cert_ocsp();
178
179  void set_tls_ticket_key_memcached_dispatcher(
180      std::unique_ptr<MemcachedDispatcher> dispatcher);
181
182  MemcachedDispatcher *get_tls_ticket_key_memcached_dispatcher() const;
183  void on_tls_ticket_key_network_error(ev_timer *w);
184  void on_tls_ticket_key_not_found(ev_timer *w);
185  void
186  on_tls_ticket_key_get_success(const std::shared_ptr<TicketKeys> &ticket_keys,
187                                ev_timer *w);
188  void schedule_next_tls_ticket_key_memcached_get(ev_timer *w);
189  SSL_CTX *create_tls_ticket_key_memcached_ssl_ctx();
190  // Returns the SSL_CTX at all_ssl_ctx_[idx].  This does not perform
191  // array bound checking.
192  SSL_CTX *get_ssl_ctx(size_t idx) const;
193
194  const std::vector<SSL_CTX *> &get_indexed_ssl_ctx(size_t idx) const;
195#ifdef ENABLE_HTTP3
196  const std::vector<SSL_CTX *> &get_quic_indexed_ssl_ctx(size_t idx) const;
197
198  int forward_quic_packet(const UpstreamAddr *faddr, const Address &remote_addr,
199                          const Address &local_addr, const ngtcp2_pkt_info &pi,
200                          const uint8_t *cid_prefix, const uint8_t *data,
201                          size_t datalen);
202
203  void set_quic_keying_materials(std::shared_ptr<QUICKeyingMaterials> qkms);
204  const std::shared_ptr<QUICKeyingMaterials> &get_quic_keying_materials() const;
205
206  void set_cid_prefixes(
207      const std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>>
208          &cid_prefixes);
209
210  void set_quic_lingering_worker_processes(
211      const std::vector<QUICLingeringWorkerProcess> &quic_lwps);
212
213  // Return matching QUICLingeringWorkerProcess which has a CID prefix
214  // such that |dcid| starts with it.  If no such
215  // QUICLingeringWorkerProcess, it returns nullptr.
216  QUICLingeringWorkerProcess *
217  match_quic_lingering_worker_process_cid_prefix(const uint8_t *dcid,
218                                                 size_t dcidlen);
219
220  int forward_quic_packet_to_lingering_worker_process(
221      QUICLingeringWorkerProcess *quic_lwp, const Address &remote_addr,
222      const Address &local_addr, const ngtcp2_pkt_info &pi, const uint8_t *data,
223      size_t datalen);
224
225  void set_quic_ipc_fd(int fd);
226
227  int quic_ipc_read();
228
229#  ifdef HAVE_LIBBPF
230  std::vector<BPFRef> &get_quic_bpf_refs();
231  void unload_bpf_objects();
232#  endif // HAVE_LIBBPF
233#endif   // ENABLE_HTTP3
234
235#ifdef HAVE_NEVERBLEED
236  void set_neverbleed(neverbleed_t *nb);
237#endif // HAVE_NEVERBLEED
238
239  // Send SerialEvent SerialEventType::REPLACE_DOWNSTREAM to this
240  // object.
241  void send_replace_downstream(
242      const std::shared_ptr<DownstreamConfig> &downstreamconf);
243  // Internal function to send |ev| to this object.
244  void send_serial_event(SerialEvent ev);
245  // Handles SerialEvents received.
246  void handle_serial_event();
247  // Sends WorkerEvent to make them replace downstream.
248  void
249  worker_replace_downstream(std::shared_ptr<DownstreamConfig> downstreamconf);
250
251  void set_enable_acceptor_on_ocsp_completion(bool f);
252
253private:
254  // Stores all SSL_CTX objects.
255  std::vector<SSL_CTX *> all_ssl_ctx_;
256  // Stores all SSL_CTX objects in a way that its index is stored in
257  // cert_tree.  The SSL_CTXs stored in the same index share the same
258  // hostname, but could have different signature algorithm.  The
259  // selection among them are performed by hostname presented by SNI,
260  // and signature algorithm presented by client.
261  std::vector<std::vector<SSL_CTX *>> indexed_ssl_ctx_;
262#ifdef ENABLE_HTTP3
263  std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>> cid_prefixes_;
264  std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>>
265      lingering_cid_prefixes_;
266  int quic_ipc_fd_;
267  std::vector<QUICLingeringWorkerProcess> quic_lingering_worker_processes_;
268#  ifdef HAVE_LIBBPF
269  std::vector<BPFRef> quic_bpf_refs_;
270#  endif // HAVE_LIBBPF
271  std::shared_ptr<QUICKeyingMaterials> quic_keying_materials_;
272  std::vector<SSL_CTX *> quic_all_ssl_ctx_;
273  std::vector<std::vector<SSL_CTX *>> quic_indexed_ssl_ctx_;
274#endif // ENABLE_HTTP3
275  OCSPUpdateContext ocsp_;
276  std::mt19937 &gen_;
277  // ev_loop for each worker
278  std::vector<struct ev_loop *> worker_loops_;
279  // Worker instances when multi threaded mode (-nN, N >= 2) is used.
280  // If at least one frontend enables API request, we allocate 1
281  // additional worker dedicated to API request .
282  std::vector<std::unique_ptr<Worker>> workers_;
283  // mutex for serial event resive buffer handling
284  std::mutex serial_event_mu_;
285  // SerialEvent receive buffer
286  std::vector<SerialEvent> serial_events_;
287  // Worker instance used when single threaded mode (-n1) is used.
288  // Otherwise, nullptr and workers_ has instances of Worker instead.
289  std::unique_ptr<Worker> single_worker_;
290  std::unique_ptr<tls::CertLookupTree> cert_tree_;
291#ifdef ENABLE_HTTP3
292  std::unique_ptr<tls::CertLookupTree> quic_cert_tree_;
293#endif // ENABLE_HTTP3
294  std::unique_ptr<MemcachedDispatcher> tls_ticket_key_memcached_dispatcher_;
295  // Current TLS session ticket keys.  Note that TLS connection does
296  // not refer to this field directly.  They use TicketKeys object in
297  // Worker object.
298  std::shared_ptr<TicketKeys> ticket_keys_;
299  struct ev_loop *loop_;
300  std::vector<std::unique_ptr<AcceptHandler>> acceptors_;
301#ifdef HAVE_NEVERBLEED
302  neverbleed_t *nb_;
303#endif // HAVE_NEVERBLEED
304  ev_timer disable_acceptor_timer_;
305  ev_timer ocsp_timer_;
306  ev_async thread_join_asyncev_;
307  ev_async serial_event_asyncev_;
308#ifndef NOTHREADS
309  std::future<void> thread_join_fut_;
310#endif // NOTHREADS
311  size_t tls_ticket_key_memcached_get_retry_count_;
312  size_t tls_ticket_key_memcached_fail_count_;
313  unsigned int worker_round_robin_cnt_;
314  bool graceful_shutdown_;
315  // true if acceptors should be enabled after the initial ocsp update
316  // has finished.
317  bool enable_acceptor_on_ocsp_completion_;
318};
319
320} // namespace shrpx
321
322#endif // SHRPX_CONNECTION_HANDLER_H
323