1/* 2 * lws-minimal-http-client-multi 3 * 4 * Written in 2010-2021 by Andy Green <andy@warmcat.com> 5 * 6 * This file is made available under the Creative Commons CC0 1.0 7 * Universal Public Domain Dedication. 8 * 9 * This demonstrates the a minimal http client using lws, which makes 10 * 8 downloads simultaneously from warmcat.com. 11 * 12 * Currently that takes the form of 8 individual simultaneous tcp and 13 * tls connections, which happen concurrently. Notice that the ordering 14 * of the returned payload may be intermingled for the various connections. 15 * 16 * By default the connections happen all together at the beginning and operate 17 * concurrently, which is fast. However this is resource-intenstive, there are 18 * 8 tcp connections, 8 tls tunnels on both the client and server. You can 19 * instead opt to have the connections happen one after the other inside a 20 * single tcp connection and tls tunnel, using HTTP/1.1 pipelining. To be 21 * eligible to be pipelined on another existing connection to the same server, 22 * the client connection must have the LCCSCF_PIPELINE flag on its 23 * info.ssl_connection member (this is independent of whether the connection 24 * is in ssl mode or not). 25 * 26 * HTTP/1.0: Pipelining only possible if Keep-Alive: yes sent by server 27 * HTTP/1.1: always possible... serializes requests 28 * HTTP/2: always possible... all requests sent as individual streams in parallel 29 * 30 * Note: stats are kept on tls session reuse and checked depending on mode 31 * 32 * - default: no reuse expected (connections made too quickly at once) 33 * - staggered, no pipeline: n - 1 reuse expected 34 * - staggered, pipelined: no reuse expected 35 */ 36 37#include <libwebsockets.h> 38#include <string.h> 39#include <signal.h> 40#include <assert.h> 41#include <time.h> 42#if !defined(WIN32) 43#include <sys/stat.h> 44#include <fcntl.h> 45#include <unistd.h> 46#endif 47 48#define COUNT 8 49 50struct cliuser { 51 int index; 52}; 53 54static int completed, failed, numbered, stagger_idx, posting, count = COUNT, 55#if defined(LWS_WITH_TLS_SESSIONS) 56 reuse, 57#endif 58 staggered; 59static lws_sorted_usec_list_t sul_stagger; 60static struct lws_client_connect_info i; 61static struct lws *client_wsi[COUNT]; 62static char urlpath[64], intr; 63static struct lws_context *context; 64 65/* we only need this for tracking POST emit state */ 66 67struct pss { 68 char body_part; 69}; 70 71#if defined(LWS_WITH_TLS_SESSIONS) && !defined(LWS_WITH_MBEDTLS) && !defined(WIN32) 72 73/* this should work OK on win32, but not adapted for non-posix file apis */ 74 75static int 76sess_save_cb(struct lws_context *cx, struct lws_tls_session_dump *info) 77{ 78 char path[128]; 79 int fd, n; 80 81 lws_snprintf(path, sizeof(path), "%s/lws_tls_sess_%s", (const char *)info->opaque, 82 info->tag); 83 fd = open(path, LWS_O_WRONLY | O_CREAT | O_TRUNC, 0600); 84 if (fd < 0) { 85 lwsl_warn("%s: cannot open %s\n", __func__, path); 86 return 1; 87 } 88 89 n = (int)write(fd, info->blob, info->blob_len); 90 91 close(fd); 92 93 return n != (int)info->blob_len; 94} 95 96static int 97sess_load_cb(struct lws_context *cx, struct lws_tls_session_dump *info) 98{ 99 struct stat sta; 100 char path[128]; 101 int fd, n; 102 103 lws_snprintf(path, sizeof(path), "%s/lws_tls_sess_%s", (const char *)info->opaque, 104 info->tag); 105 fd = open(path, LWS_O_RDONLY); 106 if (fd < 0) 107 return 1; 108 109 if (fstat(fd, &sta) || !sta.st_size) 110 goto bail; 111 112 info->blob = malloc((size_t)sta.st_size); 113 /* caller will free this */ 114 if (!info->blob) 115 goto bail; 116 117 info->blob_len = (size_t)sta.st_size; 118 119 n = (int)read(fd, info->blob, info->blob_len); 120 close(fd); 121 122 return n != (int)info->blob_len; 123 124bail: 125 close(fd); 126 127 return 1; 128} 129#endif 130 131#if defined(LWS_WITH_CONMON) 132void 133dump_conmon_data(struct lws *wsi) 134{ 135 const struct addrinfo *ai; 136 struct lws_conmon cm; 137 char ads[48]; 138 139 lws_conmon_wsi_take(wsi, &cm); 140 141 lws_sa46_write_numeric_address(&cm.peer46, ads, sizeof(ads)); 142 lwsl_notice("%s: peer %s, dns: %uus, sockconn: %uus, tls: %uus, txn_resp: %uus\n", 143 __func__, ads, 144 (unsigned int)cm.ciu_dns, 145 (unsigned int)cm.ciu_sockconn, 146 (unsigned int)cm.ciu_tls, 147 (unsigned int)cm.ciu_txn_resp); 148 149 ai = cm.dns_results_copy; 150 while (ai) { 151 lws_sa46_write_numeric_address((lws_sockaddr46 *)ai->ai_addr, ads, sizeof(ads)); 152 lwsl_notice("%s: DNS %s\n", __func__, ads); 153 ai = ai->ai_next; 154 } 155 156 /* 157 * This destroys the DNS list in the lws_conmon that we took 158 * responsibility for when we used lws_conmon_wsi_take() 159 */ 160 161 lws_conmon_release(&cm); 162} 163#endif 164 165static int 166callback_http(struct lws *wsi, enum lws_callback_reasons reason, 167 void *user, void *in, size_t len) 168{ 169 char buf[LWS_PRE + 1024], *start = &buf[LWS_PRE], *p = start, 170 *end = &buf[sizeof(buf) - 1]; 171 int n, idx = (int)(intptr_t)lws_get_opaque_user_data(wsi); 172 struct pss *pss = (struct pss *)user; 173 174 switch (reason) { 175 176 case LWS_CALLBACK_ESTABLISHED_CLIENT_HTTP: 177 lwsl_user("LWS_CALLBACK_ESTABLISHED_CLIENT_HTTP: idx: %d, resp %u\n", 178 idx, lws_http_client_http_response(wsi)); 179 180#if defined(LWS_WITH_TLS_SESSIONS) && !defined(LWS_WITH_MBEDTLS) && !defined(WIN32) 181 if (lws_tls_session_is_reused(wsi)) 182 reuse++; 183 else 184 /* 185 * Attempt to store any new session into 186 * external storage 187 */ 188 if (lws_tls_session_dump_save(lws_get_vhost_by_name(context, "default"), 189 i.host, (uint16_t)i.port, 190 sess_save_cb, "/tmp")) 191 lwsl_warn("%s: session save failed\n", __func__); 192#endif 193 break; 194 195 /* because we are protocols[0] ... */ 196 case LWS_CALLBACK_CLIENT_CONNECTION_ERROR: 197 lwsl_err("CLIENT_CONNECTION_ERROR: %s\n", 198 in ? (char *)in : "(null)"); 199 client_wsi[idx] = NULL; 200 failed++; 201 202#if defined(LWS_WITH_CONMON) 203 dump_conmon_data(wsi); 204#endif 205 206 goto finished; 207 208 /* chunks of chunked content, with header removed */ 209 case LWS_CALLBACK_RECEIVE_CLIENT_HTTP_READ: 210 lwsl_user("RECEIVE_CLIENT_HTTP_READ: conn %d: read %d\n", idx, (int)len); 211 lwsl_hexdump_info(in, len); 212 return 0; /* don't passthru */ 213 214 case LWS_CALLBACK_CLIENT_APPEND_HANDSHAKE_HEADER: 215 216 /* 217 * Tell lws we are going to send the body next... 218 */ 219 if (posting && !lws_http_is_redirected_to_get(wsi)) { 220 lwsl_user("%s: conn %d, doing POST flow\n", __func__, idx); 221 lws_client_http_body_pending(wsi, 1); 222 lws_callback_on_writable(wsi); 223 } else 224 lwsl_user("%s: conn %d, doing GET flow\n", __func__, idx); 225 break; 226 227 /* uninterpreted http content */ 228 case LWS_CALLBACK_RECEIVE_CLIENT_HTTP: 229 { 230 char buffer[1024 + LWS_PRE]; 231 char *px = buffer + LWS_PRE; 232 int lenx = sizeof(buffer) - LWS_PRE; 233 234 if (lws_http_client_read(wsi, &px, &lenx) < 0) 235 return -1; 236 } 237 return 0; /* don't passthru */ 238 239 case LWS_CALLBACK_COMPLETED_CLIENT_HTTP: 240 lwsl_user("LWS_CALLBACK_COMPLETED_CLIENT_HTTP %s: idx %d\n", 241 lws_wsi_tag(wsi), idx); 242 client_wsi[idx] = NULL; 243 goto finished; 244 245 case LWS_CALLBACK_CLOSED_CLIENT_HTTP: 246 lwsl_info("%s: closed: %s\n", __func__, lws_wsi_tag(client_wsi[idx])); 247 248#if defined(LWS_WITH_CONMON) 249 dump_conmon_data(wsi); 250#endif 251 252 if (client_wsi[idx]) { 253 /* 254 * If it completed normally, it will have been set to 255 * NULL then already. So we are dealing with an 256 * abnormal, failing, close 257 */ 258 client_wsi[idx] = NULL; 259 failed++; 260 goto finished; 261 } 262 break; 263 264 case LWS_CALLBACK_CLIENT_HTTP_WRITEABLE: 265 if (!posting) 266 break; 267 if (lws_http_is_redirected_to_get(wsi)) 268 break; 269 lwsl_info("LWS_CALLBACK_CLIENT_HTTP_WRITEABLE: %s, idx %d," 270 " part %d\n", lws_wsi_tag(wsi), idx, pss->body_part); 271 272 n = LWS_WRITE_HTTP; 273 274 /* 275 * For a small body like this, we could prepare it in memory and 276 * send it all at once. But to show how to handle, eg, 277 * arbitrary-sized file payloads, or huge form-data fields, the 278 * sending is done in multiple passes through the event loop. 279 */ 280 281 switch (pss->body_part++) { 282 case 0: 283 if (lws_client_http_multipart(wsi, "text", NULL, NULL, 284 &p, end)) 285 return -1; 286 /* notice every usage of the boundary starts with -- */ 287 p += lws_snprintf(p, lws_ptr_diff_size_t(end, p), "my text field\xd\xa"); 288 break; 289 case 1: 290 if (lws_client_http_multipart(wsi, "file", "myfile.txt", 291 "text/plain", &p, end)) 292 return -1; 293 p += lws_snprintf(p, lws_ptr_diff_size_t(end, p), 294 "This is the contents of the " 295 "uploaded file.\xd\xa" 296 "\xd\xa"); 297 break; 298 case 2: 299 if (lws_client_http_multipart(wsi, NULL, NULL, NULL, 300 &p, end)) 301 return -1; 302 lws_client_http_body_pending(wsi, 0); 303 /* necessary to support H2, it means we will write no 304 * more on this stream */ 305 n = LWS_WRITE_HTTP_FINAL; 306 break; 307 308 default: 309 /* 310 * We can get extra callbacks here, if nothing to do, 311 * then do nothing. 312 */ 313 return 0; 314 } 315 316 if (lws_write(wsi, (uint8_t *)start, lws_ptr_diff_size_t(p, start), (enum lws_write_protocol)n) 317 != lws_ptr_diff(p, start)) 318 return 1; 319 320 if (n != LWS_WRITE_HTTP_FINAL) 321 lws_callback_on_writable(wsi); 322 323 break; 324 325 default: 326 break; 327 } 328 329 return lws_callback_http_dummy(wsi, reason, user, in, len); 330 331finished: 332 if (++completed == count) { 333 if (!failed) 334 lwsl_user("Done: all OK\n"); 335 else 336 lwsl_err("Done: failed: %d\n", failed); 337 intr = 1; 338 /* 339 * This is how we can exit the event loop even when it's an 340 * event library backing it... it will start and stage the 341 * destroy to happen after we exited this service for each pt 342 */ 343 lws_context_destroy(lws_get_context(wsi)); 344 } 345 346 return 0; 347} 348 349static const struct lws_protocols protocols[] = { 350 { "http", callback_http, sizeof(struct pss), 0, 0, NULL, 0 }, 351 LWS_PROTOCOL_LIST_TERM 352}; 353 354#if defined(LWS_WITH_SYS_METRICS) 355 356static int 357my_metric_report(lws_metric_pub_t *mp) 358{ 359 lws_metric_bucket_t *sub = mp->u.hist.head; 360 char buf[192]; 361 362 do { 363 if (lws_metrics_format(mp, &sub, buf, sizeof(buf))) 364 lwsl_user("%s: %s\n", __func__, buf); 365 } while ((mp->flags & LWSMTFL_REPORT_HIST) && sub); 366 367 /* 0 = leave metric to accumulate, 1 = reset the metric */ 368 369 return 1; 370} 371 372static const lws_system_ops_t system_ops = { 373 .metric_report = my_metric_report, 374}; 375 376#endif 377 378static void 379stagger_cb(lws_sorted_usec_list_t *sul); 380 381static void 382lws_try_client_connection(struct lws_client_connect_info *i, int m) 383{ 384 char path[128]; 385 386 if (numbered) { 387 lws_snprintf(path, sizeof(path), "/%d.png", m + 1); 388 i->path = path; 389 } else 390 i->path = urlpath; 391 392 i->pwsi = &client_wsi[m]; 393 i->opaque_user_data = (void *)(intptr_t)m; 394 395 if (!lws_client_connect_via_info(i)) { 396 failed++; 397 lwsl_user("%s: failed: conn idx %d\n", __func__, m); 398 if (++completed == count) { 399 lwsl_user("Done: failed: %d\n", failed); 400 lws_context_destroy(context); 401 } 402 } else 403 lwsl_user("started connection %s: idx %d (%s)\n", 404 lws_wsi_tag(client_wsi[m]), m, i->path); 405} 406 407 408static int 409system_notify_cb(lws_state_manager_t *mgr, lws_state_notify_link_t *link, 410 int current, int target) 411{ 412 struct lws_context *context = mgr->parent; 413 int m; 414 415 if (current != LWS_SYSTATE_OPERATIONAL || target != LWS_SYSTATE_OPERATIONAL) 416 return 0; 417 418 /* all the system prerequisites are ready */ 419 420 if (!staggered) 421 /* 422 * just pile on all the connections at once, testing the 423 * pipeline queuing before the first is connected 424 */ 425 for (m = 0; m < count; m++) 426 lws_try_client_connection(&i, m); 427 else 428 /* 429 * delay the connections slightly 430 */ 431 lws_sul_schedule(context, 0, &sul_stagger, stagger_cb, 432 50 * LWS_US_PER_MS); 433 434 return 0; 435} 436 437static void 438signal_cb(void *handle, int signum) 439{ 440 switch (signum) { 441 case SIGTERM: 442 case SIGINT: 443 break; 444 default: 445 lwsl_err("%s: signal %d\n", __func__, signum); 446 break; 447 } 448 lws_context_destroy(context); 449} 450 451static void 452sigint_handler(int sig) 453{ 454 signal_cb(NULL, sig); 455} 456 457#if defined(WIN32) 458int gettimeofday(struct timeval * tp, struct timezone * tzp) 459{ 460 // Note: some broken versions only have 8 trailing zero's, the correct epoch has 9 trailing zero's 461 // This magic number is the number of 100 nanosecond intervals since January 1, 1601 (UTC) 462 // until 00:00:00 January 1, 1970 463 static const uint64_t EPOCH = ((uint64_t) 116444736000000000ULL); 464 465 SYSTEMTIME system_time; 466 FILETIME file_time; 467 uint64_t time; 468 469 GetSystemTime( &system_time ); 470 SystemTimeToFileTime( &system_time, &file_time ); 471 time = ((uint64_t)file_time.dwLowDateTime ) ; 472 time += ((uint64_t)file_time.dwHighDateTime) << 32; 473 474 tp->tv_sec = (long) ((time - EPOCH) / 10000000L); 475 tp->tv_usec = (long) (system_time.wMilliseconds * 1000); 476 return 0; 477} 478#endif 479 480unsigned long long us(void) 481{ 482 struct timeval t; 483 484 gettimeofday(&t, NULL); 485 486 return ((unsigned long long)t.tv_sec * 1000000ull) + (unsigned long long)t.tv_usec; 487} 488 489static void 490stagger_cb(lws_sorted_usec_list_t *sul) 491{ 492 lws_usec_t next; 493 494 /* 495 * open the connections at 100ms intervals, with the 496 * last one being after 1s, testing both queuing, and 497 * direct H2 stream addition stability 498 */ 499 lws_try_client_connection(&i, stagger_idx++); 500 501 if (stagger_idx == count) 502 return; 503 504 next = 150 * LWS_US_PER_MS; 505 if (stagger_idx == count - 1) 506 next += 400 * LWS_US_PER_MS; 507 508#if defined(LWS_WITH_TLS_SESSIONS) 509 if (stagger_idx == 1) 510 next += 600 * LWS_US_PER_MS; 511#endif 512 513 lws_sul_schedule(context, 0, &sul_stagger, stagger_cb, next); 514} 515 516int main(int argc, const char **argv) 517{ 518 lws_state_notify_link_t notifier = { { NULL, NULL, NULL }, 519 system_notify_cb, "app" }; 520 lws_state_notify_link_t *na[] = { ¬ifier, NULL }; 521 struct lws_context_creation_info info; 522 unsigned long long start; 523 const char *p; 524#if defined(LWS_WITH_TLS_SESSIONS) 525 int pl = 0; 526#endif 527 528 memset(&info, 0, sizeof info); /* otherwise uninitialized garbage */ 529 memset(&i, 0, sizeof i); /* otherwise uninitialized garbage */ 530 531 lws_cmdline_option_handle_builtin(argc, argv, &info); 532 533 info.signal_cb = signal_cb; 534 info.options = LWS_SERVER_OPTION_DO_SSL_GLOBAL_INIT; 535 536 if (lws_cmdline_option(argc, argv, "--uv")) 537 info.options |= LWS_SERVER_OPTION_LIBUV; 538 else 539 if (lws_cmdline_option(argc, argv, "--event")) 540 info.options |= LWS_SERVER_OPTION_LIBEVENT; 541 else 542 if (lws_cmdline_option(argc, argv, "--ev")) 543 info.options |= LWS_SERVER_OPTION_LIBEV; 544 else 545 if (lws_cmdline_option(argc, argv, "--glib")) 546 info.options |= LWS_SERVER_OPTION_GLIB; 547 else 548 signal(SIGINT, sigint_handler); 549 550 staggered = !!lws_cmdline_option(argc, argv, "-s"); 551 552 lwsl_user("LWS minimal http client [-s (staggered)] [-p (pipeline)]\n"); 553 lwsl_user(" [--h1 (http/1 only)] [-l (localhost)] [-d <logs>]\n"); 554 lwsl_user(" [-n (numbered)] [--post]\n"); 555 556 info.port = CONTEXT_PORT_NO_LISTEN; /* we do not run any server */ 557 info.protocols = protocols; 558 /* 559 * since we know this lws context is only ever going to be used with 560 * COUNT client wsis / fds / sockets at a time, let lws know it doesn't 561 * have to use the default allocations for fd tables up to ulimit -n. 562 * It will just allocate for 1 internal and COUNT + 1 (allowing for h2 563 * network wsi) that we will use. 564 */ 565 info.fd_limit_per_thread = 1 + COUNT + 1; 566 info.register_notifier_list = na; 567 info.pcontext = &context; 568 569#if defined(LWS_WITH_SYS_METRICS) 570 info.system_ops = &system_ops; 571#endif 572 573#if defined(LWS_WITH_MBEDTLS) || defined(USE_WOLFSSL) 574 /* 575 * OpenSSL uses the system trust store. mbedTLS has to be told which 576 * CA to trust explicitly. 577 */ 578 info.client_ssl_ca_filepath = "./warmcat.com.cer"; 579#endif 580 581 /* vhost option allowing tls session reuse, requires 582 * LWS_WITH_TLS_SESSIONS build option */ 583 if (lws_cmdline_option(argc, argv, "--no-tls-session-reuse")) 584 info.options |= LWS_SERVER_OPTION_DISABLE_TLS_SESSION_CACHE; 585 586 if ((p = lws_cmdline_option(argc, argv, "--limit"))) 587 info.simultaneous_ssl_restriction = atoi(p); 588 589 if ((p = lws_cmdline_option(argc, argv, "--ssl-handshake-serialize"))) 590 /* We only consider simultaneous_ssl_restriction > 1 use cases. 591 * If ssl isn't limited or only 1 is allowed, we don't care. 592 */ 593 info.simultaneous_ssl_handshake_restriction = atoi(p); 594 595 context = lws_create_context(&info); 596 if (!context) { 597 lwsl_err("lws init failed\n"); 598 return 1; 599 } 600 601#if defined(LWS_ROLE_H2) && defined(LWS_ROLE_H1) 602 i.alpn = "h2,http/1.1"; 603#elif defined(LWS_ROLE_H2) 604 i.alpn = "h2"; 605#elif defined(LWS_ROLE_H1) 606 i.alpn = "http/1.1"; 607#endif 608 609 i.context = context; 610 i.ssl_connection = LCCSCF_USE_SSL | 611 LCCSCF_H2_QUIRK_OVERFLOWS_TXCR | 612 LCCSCF_H2_QUIRK_NGHTTP2_END_STREAM; 613 614 if (lws_cmdline_option(argc, argv, "--post")) { 615 posting = 1; 616 i.method = "POST"; 617 i.ssl_connection |= LCCSCF_HTTP_MULTIPART_MIME; 618 } else 619 i.method = "GET"; 620 621 /* enables h1 or h2 connection sharing */ 622 if (lws_cmdline_option(argc, argv, "-p")) { 623 i.ssl_connection |= LCCSCF_PIPELINE; 624#if defined(LWS_WITH_TLS_SESSIONS) 625 pl = 1; 626#endif 627 } 628 629#if defined(LWS_WITH_CONMON) 630 if (lws_cmdline_option(argc, argv, "--conmon")) 631 i.ssl_connection |= LCCSCF_CONMON; 632#endif 633 634 /* force h1 even if h2 available */ 635 if (lws_cmdline_option(argc, argv, "--h1")) 636 i.alpn = "http/1.1"; 637 638 strcpy(urlpath, "/"); 639 640 if (lws_cmdline_option(argc, argv, "-l")) { 641 i.port = 7681; 642 i.address = "localhost"; 643 i.ssl_connection |= LCCSCF_ALLOW_SELFSIGNED; 644 if (posting) 645 strcpy(urlpath, "/formtest"); 646 } else { 647 i.port = 443; 648 i.address = "libwebsockets.org"; 649 if (posting) 650 strcpy(urlpath, "/testserver/formtest"); 651 } 652 653 if (lws_cmdline_option(argc, argv, "--no-tls")) 654 i.ssl_connection &= ~(LCCSCF_USE_SSL); 655 656 if (lws_cmdline_option(argc, argv, "-n")) 657 numbered = 1; 658 659 if ((p = lws_cmdline_option(argc, argv, "--server"))) 660 i.address = p; 661 662 if ((p = lws_cmdline_option(argc, argv, "--port"))) 663 i.port = atoi(p); 664 665 if ((p = lws_cmdline_option(argc, argv, "--path"))) 666 lws_strncpy(urlpath, p, sizeof(urlpath)); 667 668 if ((p = lws_cmdline_option(argc, argv, "-c"))) 669 if (atoi(p) <= COUNT && atoi(p)) 670 count = atoi(p); 671 672 i.host = i.address; 673 i.origin = i.address; 674 i.protocol = protocols[0].name; 675 676#if defined(LWS_WITH_TLS_SESSIONS) && !defined(LWS_WITH_MBEDTLS) && !defined(WIN32) 677 /* 678 * Attempt to preload a session from external storage 679 */ 680 if (lws_tls_session_dump_load(lws_get_vhost_by_name(context, "default"), 681 i.host, (uint16_t)i.port, sess_load_cb, "/tmp")) 682 lwsl_warn("%s: session load failed\n", __func__); 683#endif 684 685 start = us(); 686 while (!intr && !lws_service(context, 0)) 687 ; 688 689#if defined(LWS_WITH_TLS_SESSIONS) 690 lwsl_user("%s: session reuse count %d\n", __func__, reuse); 691 692 if (staggered && !pl && !reuse) { 693 lwsl_err("%s: failing, expected 1 .. %d reused\n", __func__, count - 1); 694 // too difficult to reproduce in CI 695 // failed = 1; 696 } 697#endif 698 699 lwsl_user("Duration: %lldms\n", (us() - start) / 1000); 700 lws_context_destroy(context); 701 702 lwsl_user("Exiting with %d\n", failed || completed != count); 703 704 return failed || completed != count; 705} 706