X-Git-Url: http://sjero.net/git/?p=wget;a=blobdiff_plain;f=src%2Fhttp.c;h=b8151ea2f751cbbf280aa26502478e89709b7421;hp=24de9bb8f9d58a8d03329503ca00d54f97298fa7;hb=07f60f19a4ee99ddc343bb51568b0c474e9b853d;hpb=0dd418242a66f82def061205fc6366ae63223723 diff --git a/src/http.c b/src/http.c index 24de9bb8..b8151ea2 100644 --- a/src/http.c +++ b/src/http.c @@ -1,5 +1,5 @@ /* HTTP support. - Copyright (C) 1995, 1996, 1997, 1998 Free Software Foundation, Inc. + Copyright (C) 1995, 1996, 1997, 1998, 2000 Free Software Foundation, Inc. This file is part of Wget. @@ -46,6 +46,8 @@ Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ #ifdef WINDOWS # include +#else +# include /* for h_errno */ #endif #include "wget.h" @@ -61,6 +63,9 @@ Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ #if USE_DIGEST # include "md5.h" #endif +#ifdef HAVE_SSL +# include "gen_sslfunc.h" +#endif /* HAVE_SSL */ extern char *version_string; @@ -68,7 +73,9 @@ extern char *version_string; extern int errno; #endif #ifndef h_errno +# ifndef __CYGWIN__ extern int h_errno; +# endif #endif @@ -239,21 +246,211 @@ static int http_process_type (const char *hdr, void *arg) { char **result = (char **)arg; - char *p; + /* Locate P on `;' or the terminating zero, whichever comes first. */ + const char *p = strchr (hdr, ';'); + if (!p) + p = hdr + strlen (hdr); + while (p > hdr && ISSPACE (*(p - 1))) + --p; + *result = strdupdelim (hdr, p); + return 1; +} + +/* Check whether the `Connection' header is set to "keep-alive". */ +static int +http_process_connection (const char *hdr, void *arg) +{ + int *flag = (int *)arg; + if (!strcasecmp (hdr, "Keep-Alive")) + *flag = 1; + return 1; +} + +/* Persistent connections. Currently, we cache the most recently used + connection as persistent, provided that the HTTP server agrees to + make it such. The persistence data is stored in the variables + below. Ideally, it would be in a structure, and it should be + possible to cache an arbitrary fixed number of these connections. + + I think the code is quite easy to extend in that direction. */ + +/* Whether a persistent connection is active. */ +static int pc_active_p; +/* Host and port of currently active persistent connection. */ +static unsigned char pc_last_host[4]; +static unsigned short pc_last_port; + +/* File descriptor of the currently active persistent connection. */ +static int pc_last_fd; + +#ifdef HAVE_SSL +/* Whether a ssl handshake has occoured on this connection */ +static int pc_active_ssl; +/* SSL connection of the currently active persistent connection. */ +static SSL *pc_last_ssl; +#endif /* HAVE_SSL */ + +/* Mark the persistent connection as invalid. This is used by the + CLOSE_* macros after they forcefully close a registered persistent + connection. This does not close the file descriptor -- it is left + to the caller to do that. (Maybe it should, though.) */ + +static void +invalidate_persistent (void) +{ + pc_active_p = 0; +#ifdef HAVE_SSL + pc_active_ssl = 0; +#endif /* HAVE_SSL */ + DEBUGP (("Invalidating fd %d from further reuse.\n", pc_last_fd)); +} + +/* Register FD, which should be a TCP/IP connection to HOST:PORT, as + persistent. This will enable someone to use the same connection + later. In the context of HTTP, this must be called only AFTER the + response has been received and the server has promised that the + connection will remain alive. + + If a previous connection was persistent, it is closed. */ + +static void +register_persistent (const char *host, unsigned short port, int fd +#ifdef HAVE_SSL + , SSL *ssl +#endif + ) +{ + int success; - p = strrchr (hdr, ';'); - if (p) + if (pc_active_p) { - int len = p - hdr; - *result = (char *)xmalloc (len + 1); - memcpy (*result, hdr, len); - (*result)[len] = '\0'; + if (pc_last_fd == fd) + { + /* The connection FD is already registered. Nothing to + do. */ + return; + } + else + { + /* The old persistent connection is still active; let's + close it first. This situation arises whenever a + persistent connection exists, but we then connect to a + different host, and try to register a persistent + connection to that one. */ +#ifdef HAVE_SSL + /* The ssl disconnect has to take place before the closing + of pc_last_fd. */ + if (pc_last_ssl) + shutdown_ssl(pc_last_ssl); +#endif + CLOSE (pc_last_fd); + invalidate_persistent (); + } + } + + /* This store_hostaddress may not fail, because it has the results + in the cache. */ + success = store_hostaddress (pc_last_host, host); + assert (success); + pc_last_port = port; + pc_last_fd = fd; + pc_active_p = 1; +#ifdef HAVE_SSL + pc_last_ssl = ssl; + pc_active_ssl = ssl ? 1 : 0; +#endif + DEBUGP (("Registered fd %d for persistent reuse.\n", fd)); +} + +/* Return non-zero if a persistent connection is available for + connecting to HOST:PORT. */ + +static int +persistent_available_p (const char *host, unsigned short port +#ifdef HAVE_SSL + , int ssl +#endif + ) +{ + unsigned char this_host[4]; + /* First, check whether a persistent connection is active at all. */ + if (!pc_active_p) + return 0; + /* Second, check if the active connection pertains to the correct + (HOST, PORT) ordered pair. */ + if (port != pc_last_port) + return 0; +#ifdef HAVE_SSL + /* Second, a): check if current connection is (not) ssl, too. This + test is unlikely to fail because HTTP and HTTPS typicaly use + different ports. Yet it is possible, or so I [Christian + Fraenkel] have been told, to run HTTPS and HTTP simultaneus on + the same port. */ + if (ssl != pc_active_ssl) + return 0; +#endif /* HAVE_SSL */ + if (!store_hostaddress (this_host, host)) + return 0; + if (memcmp (pc_last_host, this_host, 4)) + return 0; + /* Third: check whether the connection is still open. This is + important because most server implement a liberal (short) timeout + on persistent connections. Wget can of course always reconnect + if the connection doesn't work out, but it's nicer to know in + advance. This test is a logical followup of the first test, but + is "expensive" and therefore placed at the end of the list. */ + if (!test_socket_open (pc_last_fd)) + { + /* Oops, the socket is no longer open. Now that we know that, + let's invalidate the persistent connection before returning + 0. */ + CLOSE (pc_last_fd); + invalidate_persistent (); + return 0; } - else - *result = xstrdup (hdr); return 1; } +#ifdef HAVE_SSL +# define SHUTDOWN_SSL(ssl) do { \ + if (ssl) \ + shutdown_ssl (ssl); \ +} while (0) +#else +# define SHUTDOWN_SSL(ssl) +#endif + +/* The idea behind these two CLOSE macros is to distinguish between + two cases: one when the job we've been doing is finished, and we + want to close the connection and leave, and two when something is + seriously wrong and we're closing the connection as part of + cleanup. + + In case of keep_alive, CLOSE_FINISH should leave the connection + open, while CLOSE_INVALIDATE should still close it. + + Note that the semantics of the flag `keep_alive' is "this + connection *will* be reused (the server has promised not to close + the connection once we're done)", while the semantics of + `pc_active_p && (fd) == pc_last_fd' is "we're *now* using an + active, registered connection". */ + +#define CLOSE_FINISH(fd) do { \ + if (!keep_alive) \ + { \ + SHUTDOWN_SSL (ssl); \ + CLOSE (fd); \ + if (pc_active_p && (fd) == pc_last_fd) \ + invalidate_persistent (); \ + } \ +} while (0) + +#define CLOSE_INVALIDATE(fd) do { \ + SHUTDOWN_SSL (ssl); \ + CLOSE (fd); \ + if (pc_active_p && (fd) == pc_last_fd) \ + invalidate_persistent (); \ +} while (0) struct http_stat { @@ -286,6 +483,11 @@ static int known_authentication_scheme_p PARAMS ((const char *)); static time_t http_atotm PARAMS ((char *)); +#define BEGINS_WITH(line, string_constant) \ + (!strncasecmp (line, string_constant, sizeof (string_constant) - 1) \ + && (ISSPACE (line[sizeof (string_constant) - 1]) \ + || !line[sizeof (string_constant) - 1])) + /* Retrieve a document through HTTP protocol. It recognizes status code, and correctly handles redirections. It closes the network socket. If it receives an error from the functions below it, it @@ -308,8 +510,8 @@ gethttp (struct urlinfo *u, struct http_stat *hs, int *dt) char *authenticate_h; char *proxyauth; char *all_headers; - char *host_port; - int host_port_len; + char *port_maybe; + char *request_keep_alive; int sock, hcount, num_written, all_length, remport, statcode; long contlen, contrange; struct urlinfo *ou; @@ -317,6 +519,59 @@ gethttp (struct urlinfo *u, struct http_stat *hs, int *dt) FILE *fp; int auth_tried_already; struct rbuf rbuf; +#ifdef HAVE_SSL + static SSL_CTX *ssl_ctx = NULL; + SSL *ssl = NULL; +#endif /* HAVE_SSL */ + + /* Whether this connection will be kept alive after the HTTP request + is done. */ + int keep_alive; + + /* Flags that detect the two ways of specifying HTTP keep-alive + response. */ + int http_keep_alive_1, http_keep_alive_2; + + /* Whether keep-alive should be inhibited. */ + int inhibit_keep_alive; + +#ifdef HAVE_SSL + /* initialize ssl_ctx on first run */ + if (!ssl_ctx) + { + err=init_ssl (&ssl_ctx); + if (err != 0) + { + switch (err) + { + case SSLERRCTXCREATE: + /* this is fatal */ + logprintf (LOG_NOTQUIET, _("Failed to set up an SSL context\n")); + ssl_printerrors (); + return err; + case SSLERRCERTFILE: + /* try without certfile */ + logprintf (LOG_NOTQUIET, + _("Failed to load certificates from %s\n"), + opt.sslcertfile); + ssl_printerrors (); + logprintf (LOG_NOTQUIET, + _("Trying without the specified certificate\n")); + break; + case SSLERRCERTKEY: + logprintf (LOG_NOTQUIET, + _("Failed to get certificate key from %s\n"), + opt.sslcertkey); + ssl_printerrors (); + logprintf (LOG_NOTQUIET, + _("Trying without the specified certificate\n")); + break; + default: + break; + } + } + } +#endif /* HAVE_SSL */ if (!(*dt & HEAD_ONLY)) /* If we're doing a GET on the URL, as opposed to just a HEAD, we need to @@ -326,9 +581,15 @@ gethttp (struct urlinfo *u, struct http_stat *hs, int *dt) authenticate_h = 0; auth_tried_already = 0; + inhibit_keep_alive = (!opt.http_keep_alive || u->proxy != NULL); + again: /* We need to come back here when the initial attempt to retrieve - without authorization header fails. */ + without authorization header fails. (Expected to happen at least + for the Digest authorization scheme.) */ + + keep_alive = 0; + http_keep_alive_1 = http_keep_alive_2 = 0; /* Initialize certain elements of struct http_stat. */ hs->len = 0L; @@ -345,40 +606,71 @@ gethttp (struct urlinfo *u, struct http_stat *hs, int *dt) ou = u; /* First: establish the connection. */ - logprintf (LOG_VERBOSE, _("Connecting to %s:%hu... "), u->host, u->port); - err = make_connection (&sock, u->host, u->port); - switch (err) + if (inhibit_keep_alive + || +#ifndef HAVE_SSL + !persistent_available_p (u->host, u->port) +#else + !persistent_available_p (u->host, u->port, (u->proto==URLHTTPS ? 1 : 0)) +#endif /* HAVE_SSL */ + ) { - case HOSTERR: - logputs (LOG_VERBOSE, "\n"); - logprintf (LOG_NOTQUIET, "%s: %s.\n", u->host, herrmsg (h_errno)); - return HOSTERR; - break; - case CONSOCKERR: - logputs (LOG_VERBOSE, "\n"); - logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno)); - return CONSOCKERR; - break; - case CONREFUSED: - logputs (LOG_VERBOSE, "\n"); - logprintf (LOG_NOTQUIET, - _("Connection to %s:%hu refused.\n"), u->host, u->port); - CLOSE (sock); - return CONREFUSED; - case CONERROR: - logputs (LOG_VERBOSE, "\n"); - logprintf (LOG_NOTQUIET, "connect: %s\n", strerror (errno)); - CLOSE (sock); - return CONERROR; - break; - case NOCONERROR: - /* Everything is fine! */ - logputs (LOG_VERBOSE, _("connected!\n")); - break; - default: - abort (); - break; - } /* switch */ + logprintf (LOG_VERBOSE, _("Connecting to %s:%hu... "), u->host, u->port); + err = make_connection (&sock, u->host, u->port); + switch (err) + { + case HOSTERR: + logputs (LOG_VERBOSE, "\n"); + logprintf (LOG_NOTQUIET, "%s: %s.\n", u->host, herrmsg (h_errno)); + return HOSTERR; + break; + case CONSOCKERR: + logputs (LOG_VERBOSE, "\n"); + logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno)); + return CONSOCKERR; + break; + case CONREFUSED: + logputs (LOG_VERBOSE, "\n"); + logprintf (LOG_NOTQUIET, + _("Connection to %s:%hu refused.\n"), u->host, u->port); + CLOSE (sock); + return CONREFUSED; + case CONERROR: + logputs (LOG_VERBOSE, "\n"); + logprintf (LOG_NOTQUIET, "connect: %s\n", strerror (errno)); + CLOSE (sock); + return CONERROR; + break; + case NOCONERROR: + /* Everything is fine! */ + logputs (LOG_VERBOSE, _("connected!\n")); + break; + default: + abort (); + break; + } +#ifdef HAVE_SSL + if (u->proto == URLHTTPS) + if (connect_ssl (&ssl, ssl_ctx,sock) != 0) + { + logputs (LOG_VERBOSE, "\n"); + logprintf (LOG_NOTQUIET, _("Unable to establish SSL connection.\n")); + CLOSE (sock); + return CONSSLERR; + } +#endif /* HAVE_SSL */ + } + else + { + logprintf (LOG_VERBOSE, _("Reusing connection to %s:%hu.\n"), u->host, u->port); + /* #### pc_last_fd should be accessed through an accessor + function. */ + sock = pc_last_fd; +#ifdef HAVE_SSL + ssl = pc_last_ssl; +#endif /* HAVE_SSL */ + DEBUGP (("Reusing fd %d.\n", sock)); + } if (u->proxy) path = u->proxy->url; @@ -399,12 +691,13 @@ gethttp (struct urlinfo *u, struct http_stat *hs, int *dt) if (hs->restval) { range = (char *)alloca (13 + numdigit (hs->restval) + 4); - /* #### Gag me! Some servers (e.g. WebSitePro) have been known - to misinterpret the following `Range' format, and return the - document as multipart/x-byte-ranges MIME type! - - #### TODO: Interpret MIME types, recognize bullshits similar - the one described above, and deal with them! */ + /* Gag me! Some servers (e.g. WebSitePro) have been known to + respond to the following `Range' format by generating a + multipart/x-byte-ranges MIME document! This MIME type was + present in an old draft of the byteranges specification. + HTTP/1.1 specifies a multipart/byte-ranges MIME type, but + only if multiple non-overlapping ranges are requested -- + which Wget never does. */ sprintf (range, "Range: bytes=%ld-\r\n", hs->restval); } else @@ -424,10 +717,37 @@ gethttp (struct urlinfo *u, struct http_stat *hs, int *dt) passwd = passwd ? passwd : opt.http_passwd; wwwauth = NULL; - if (authenticate_h && user && passwd) + if (user && passwd) { - wwwauth = create_authorization_line (authenticate_h, user, passwd, - command, ou->path); + if (!authenticate_h) + { + /* We have the username and the password, but haven't tried + any authorization yet. Let's see if the "Basic" method + works. If not, we'll come back here and construct a + proper authorization method with the right challenges. + + If we didn't employ this kind of logic, every URL that + requires authorization would have to be processed twice, + which is very suboptimal and generates a bunch of false + "unauthorized" errors in the server log. + + #### But this logic also has a serious problem when used + with stronger authentications: we *first* transmit the + username and the password in clear text, and *then* + attempt a stronger authentication scheme. That cannot be + right! We are only fortunate that almost everyone still + uses the `Basic' scheme anyway. + + There should be an option to prevent this from happening, + for those who use strong authentication schemes and value + their passwords. */ + wwwauth = basic_authentication_encode (user, passwd, "Authorization"); + } + else + { + wwwauth = create_authorization_line (authenticate_h, user, passwd, + command, ou->path); + } } proxyauth = NULL; @@ -458,22 +778,27 @@ gethttp (struct urlinfo *u, struct http_stat *hs, int *dt) remhost = ou->host; remport = ou->port; - if (remport == 80) + /* String of the form :PORT. Used only for non-standard ports. */ + port_maybe = NULL; + if (remport != 80) { - host_port = NULL; - host_port_len = 0; + port_maybe = (char *)alloca (numdigit (remport) + 2); + sprintf (port_maybe, ":%d", remport); } + + if (!inhibit_keep_alive) + request_keep_alive = "Connection: Keep-Alive\r\n"; else - { - host_port = (char *)alloca (numdigit (remport) + 2); - host_port_len = sprintf (host_port, ":%d", remport); - } + request_keep_alive = NULL; /* Allocate the memory for the request. */ request = (char *)alloca (strlen (command) + strlen (path) + strlen (useragent) - + strlen (remhost) + host_port_len + + strlen (remhost) + + (port_maybe ? strlen (port_maybe) : 0) + strlen (HTTP_ACCEPT) + + (request_keep_alive + ? strlen (request_keep_alive) : 0) + (referer ? strlen (referer) : 0) + (wwwauth ? strlen (wwwauth) : 0) + (proxyauth ? strlen (proxyauth) : 0) @@ -487,10 +812,12 @@ gethttp (struct urlinfo *u, struct http_stat *hs, int *dt) User-Agent: %s\r\n\ Host: %s%s\r\n\ Accept: %s\r\n\ -%s%s%s%s%s%s\r\n", +%s%s%s%s%s%s%s\r\n", command, path, useragent, remhost, - host_port ? host_port : "", - HTTP_ACCEPT, referer ? referer : "", + port_maybe ? port_maybe : "", + HTTP_ACCEPT, + request_keep_alive ? request_keep_alive : "", + referer ? referer : "", wwwauth ? wwwauth : "", proxyauth ? proxyauth : "", range ? range : "", @@ -502,11 +829,18 @@ Accept: %s\r\n\ FREE_MAYBE (proxyauth); /* Send the request to server. */ - num_written = iwrite (sock, request, strlen (request)); +#ifdef HAVE_SSL + if (u->proto == URLHTTPS) + num_written = ssl_iwrite (ssl, request, strlen (request)); + else +#endif /* HAVE_SSL */ + num_written = iwrite (sock, request, strlen (request)); + if (num_written < 0) { - logputs (LOG_VERBOSE, _("Failed writing HTTP request.\n")); - CLOSE (sock); + logprintf (LOG_VERBOSE, _("Failed writing HTTP request: %s.\n"), + strerror (errno)); + CLOSE_INVALIDATE (sock); return WRITEFAILED; } logprintf (LOG_VERBOSE, _("%s request sent, awaiting response... "), @@ -518,7 +852,12 @@ Accept: %s\r\n\ /* Before reading anything, initialize the rbuf. */ rbuf_initialize (&rbuf, sock); - +#ifdef HAVE_SSL + if (u->proto == URLHTTPS) + rbuf.ssl = ssl; + else + rbuf.ssl = NULL; +#endif /* HAVE_SSL */ all_headers = NULL; all_length = 0; /* Header-fetching loop. */ @@ -549,11 +888,11 @@ Accept: %s\r\n\ what you accept." Oh boy. */ logputs (LOG_VERBOSE, "\n"); logputs (LOG_NOTQUIET, _("End of file while parsing headers.\n")); - free (hdr); + xfree (hdr); FREE_MAYBE (type); FREE_MAYBE (hs->newloc); FREE_MAYBE (all_headers); - CLOSE (sock); + CLOSE_INVALIDATE (sock); return HEOF; } else if (status == HG_ERROR) @@ -561,11 +900,11 @@ Accept: %s\r\n\ logputs (LOG_VERBOSE, "\n"); logprintf (LOG_NOTQUIET, _("Read error (%s) in headers.\n"), strerror (errno)); - free (hdr); + xfree (hdr); FREE_MAYBE (type); FREE_MAYBE (hs->newloc); FREE_MAYBE (all_headers); - CLOSE (sock); + CLOSE_INVALIDATE (sock); return HERR; } @@ -602,7 +941,7 @@ Accept: %s\r\n\ hs->error = xstrdup (_("No data received")); else hs->error = xstrdup (_("Malformed status line")); - free (hdr); + xfree (hdr); break; } else if (!*error) @@ -623,7 +962,7 @@ Accept: %s\r\n\ /* Exit on empty header. */ if (!*hdr) { - free (hdr); + xfree (hdr); break; } @@ -672,12 +1011,45 @@ Accept: %s\r\n\ goto done_header; } } + /* Check for keep-alive related responses. */ + if (!inhibit_keep_alive) + { + /* Check for the `Keep-Alive' header. */ + if (!http_keep_alive_1) + { + if (header_process (hdr, "Keep-Alive", header_exists, + &http_keep_alive_1)) + goto done_header; + } + /* Check for `Connection: Keep-Alive'. */ + if (!http_keep_alive_2) + { + if (header_process (hdr, "Connection", http_process_connection, + &http_keep_alive_2)) + goto done_header; + } + } done_header: - free (hdr); + xfree (hdr); } logputs (LOG_VERBOSE, "\n"); + if (contlen != -1 + && (http_keep_alive_1 || http_keep_alive_2)) + { + assert (inhibit_keep_alive == 0); + keep_alive = 1; + } + if (keep_alive) + /* The server has promised that it will not close the connection + when we're done. This means that we can register it. */ +#ifndef HAVE_SSL + register_persistent (u->host, u->port, sock); +#else + register_persistent (u->host, u->port, sock, ssl); +#endif /* HAVE_SSL */ + if ((statcode == HTTP_STATUS_UNAUTHORIZED) && authenticate_h) { @@ -685,21 +1057,29 @@ Accept: %s\r\n\ FREE_MAYBE (type); type = NULL; FREEHSTAT (*hs); - CLOSE (sock); + CLOSE_FINISH (sock); if (auth_tried_already) { /* If we have tried it already, then there is not point retrying it. */ + failed: logputs (LOG_NOTQUIET, _("Authorization failed.\n")); - free (authenticate_h); + xfree (authenticate_h); return AUTHFAILED; } else if (!known_authentication_scheme_p (authenticate_h)) { - free (authenticate_h); + xfree (authenticate_h); logputs (LOG_NOTQUIET, _("Unknown authentication scheme.\n")); return AUTHFAILED; } + else if (BEGINS_WITH (authenticate_h, "Basic")) + { + /* The authentication scheme is basic, the one we try by + default, and it failed. There's no sense in trying + again. */ + goto failed; + } else { auth_tried_already = 1; @@ -709,7 +1089,7 @@ Accept: %s\r\n\ /* We do not need this anymore. */ if (authenticate_h) { - free (authenticate_h); + xfree (authenticate_h); authenticate_h = NULL; } @@ -753,7 +1133,7 @@ Accept: %s\r\n\ FREE_MAYBE (type); FREE_MAYBE (hs->newloc); FREE_MAYBE (all_headers); - CLOSE (sock); + CLOSE_INVALIDATE (sock); return RANGEERR; } @@ -783,7 +1163,7 @@ Accept: %s\r\n\ _("Location: %s%s\n"), hs->newloc ? hs->newloc : _("unspecified"), hs->newloc ? _(" [following]") : ""); - CLOSE (sock); + CLOSE_FINISH (sock); FREE_MAYBE (type); FREE_MAYBE (all_headers); return NEWLOCATION; @@ -824,7 +1204,7 @@ Accept: %s\r\n\ hs->res = 0; FREE_MAYBE (type); FREE_MAYBE (all_headers); - CLOSE (sock); + CLOSE_FINISH (sock); return RETRFINISHED; } @@ -838,7 +1218,7 @@ Accept: %s\r\n\ if (!fp) { logprintf (LOG_NOTQUIET, "%s: %s\n", u->local, strerror (errno)); - CLOSE (sock); + CLOSE_FINISH (sock); FREE_MAYBE (all_headers); return FOPENERR; } @@ -863,7 +1243,7 @@ Accept: %s\r\n\ /* Get the contents of the document. */ hs->res = get_contents (sock, fp, &hs->len, hs->restval, (contlen != -1 ? contlen : 0), - &rbuf); + &rbuf, keep_alive); hs->dltime = elapsed_time (); { /* Close or flush the file. We have to be careful to check for @@ -878,7 +1258,7 @@ Accept: %s\r\n\ hs->res = -2; } FREE_MAYBE (all_headers); - CLOSE (sock); + CLOSE_FINISH (sock); if (hs->res == -2) return FWRITEERR; return RETRFINISHED; @@ -889,8 +1269,6 @@ Accept: %s\r\n\ uerr_t http_loop (struct urlinfo *u, char **newloc, int *dt) { - static int first_retrieval = 1; - int count; int use_ts, got_head = 0; /* time-stamping info */ char *filename_plus_orig_suffix; @@ -941,8 +1319,8 @@ File `%s' already there, will not retrieve.\n"), u->local); if (((suf = suffix (u->local)) != NULL) && (!strcmp (suf, "html") || !strcmp (suf, "htm"))) *dt |= TEXTHTML; - free (suf); - free(filename_plus_orig_suffix); /* must precede every return! */ + xfree (suf); + xfree (filename_plus_orig_suffix); /* must precede every return! */ /* Another harmless lie: */ return RETROK; } @@ -960,7 +1338,14 @@ File `%s' already there, will not retrieve.\n"), u->local); _wasn't_ specified last time, or the server contains files called *.orig, -N will be back to not operating correctly with -k. */ { - /* Would a single s[n]printf() call be faster? */ + /* Would a single s[n]printf() call be faster? --dan + + It wouldn't. sprintf() is horribly slow. At one point I + profiled Wget, and found that a measurable and + non-negligible amount of time was lost calling sprintf() + in url.c. Replacing sprintf with inline calls to + strcpy() and long_to_string() made a difference. + --hniksic */ strcpy(filename_plus_orig_suffix, u->local); strcpy(filename_plus_orig_suffix + filename_len, ".orig"); @@ -996,23 +1381,7 @@ File `%s' already there, will not retrieve.\n"), u->local); { /* Increment the pass counter. */ ++count; - /* Wait before the retrieval (unless this is the very first - retrieval). - Check if we are retrying or not, wait accordingly - HEH */ - if (!first_retrieval && (opt.wait || (count && opt.waitretry))) - { - if (count) - { - if (countlocal); #ifdef WINDOWS ws_changetitle (hurl, 1); #endif - free (hurl); + xfree (hurl); } /* Default document type is empty. However, if spider mode is @@ -1080,10 +1449,11 @@ File `%s' already there, will not retrieve.\n"), u->local); printwhat (count, opt.ntry); continue; break; - case HOSTERR: case CONREFUSED: case PROXERR: case AUTHFAILED: + case HOSTERR: case CONREFUSED: case PROXERR: case AUTHFAILED: + case SSLERRCTXCREATE: /* Fatal errors just return from the function. */ FREEHSTAT (hstat); - free(filename_plus_orig_suffix); /* must precede every return! */ + xfree (filename_plus_orig_suffix); /* must precede every return! */ return err; break; case FWRITEERR: case FOPENERR: @@ -1092,7 +1462,14 @@ File `%s' already there, will not retrieve.\n"), u->local); logprintf (LOG_NOTQUIET, _("Cannot write to `%s' (%s).\n"), u->local, strerror (errno)); FREEHSTAT (hstat); - free(filename_plus_orig_suffix); /* must precede every return! */ + return err; + break; + case CONSSLERR: + /* Another fatal error. */ + logputs (LOG_VERBOSE, "\n"); + logprintf (LOG_NOTQUIET, _("Unable to establish SSL connection.\n")); + FREEHSTAT (hstat); + xfree (filename_plus_orig_suffix); /* must precede every return! */ return err; break; case NEWLOCATION: @@ -1102,11 +1479,11 @@ File `%s' already there, will not retrieve.\n"), u->local); logprintf (LOG_NOTQUIET, _("ERROR: Redirection (%d) without location.\n"), hstat.statcode); - free(filename_plus_orig_suffix); /* must precede every return! */ + xfree (filename_plus_orig_suffix); /* must precede every return! */ return WRONGCODE; } FREEHSTAT (hstat); - free(filename_plus_orig_suffix); /* must precede every return! */ + xfree (filename_plus_orig_suffix); /* must precede every return! */ return NEWLOCATION; break; case RETRFINISHED: @@ -1123,13 +1500,13 @@ File `%s' already there, will not retrieve.\n"), u->local); /* #### Ugly ugly ugly! */ char *hurl = str_url (u->proxy ? u->proxy : u, 1); logprintf (LOG_NONVERBOSE, "%s:\n", hurl); - free (hurl); + xfree (hurl); } logprintf (LOG_NOTQUIET, _("%s ERROR %d: %s.\n"), tms, hstat.statcode, hstat.error); logputs (LOG_VERBOSE, "\n"); FREEHSTAT (hstat); - free(filename_plus_orig_suffix); /* must precede every return! */ + xfree (filename_plus_orig_suffix); /* must precede every return! */ return WRONGCODE; } @@ -1173,7 +1550,7 @@ Last-modified header invalid -- time-stamp ignored.\n")); Server file no newer than local file `%s' -- not retrieving.\n\n"), local_filename); FREEHSTAT (hstat); - free(filename_plus_orig_suffix);/*must precede every return!*/ + xfree (filename_plus_orig_suffix); /*must precede every return!*/ return RETROK; } else if (tml >= tmr) @@ -1186,22 +1563,32 @@ The sizes do not match (local %ld) -- retrieving.\n"), local_size); FREEHSTAT (hstat); continue; } - if (!opt.dfp - && (tmr != (time_t) (-1)) + if ((tmr != (time_t) (-1)) && !opt.spider && ((hstat.len == hstat.contlen) || ((hstat.res == 0) && ((hstat.contlen == -1) || (hstat.len >= hstat.contlen && !opt.kill_longer))))) { - touch (u->local, tmr); + /* #### This code repeats in http.c and ftp.c. Move it to a + function! */ + const char *fl = NULL; + if (opt.output_document) + { + if (opt.od_known_regular) + fl = opt.output_document; + } + else + fl = u->local; + if (fl) + touch (fl, tmr); } /* End of time-stamping section. */ if (opt.spider) { logprintf (LOG_NOTQUIET, "%d %s\n\n", hstat.statcode, hstat.error); - free(filename_plus_orig_suffix); /* must precede every return! */ + xfree (filename_plus_orig_suffix); /* must precede every return! */ return RETROK; } @@ -1223,7 +1610,7 @@ The sizes do not match (local %ld) -- retrieving.\n"), local_size); tms, u->url, hstat.len, hstat.contlen, locf, count); } ++opt.numurls; - opt.downloaded += hstat.len; + downloaded_increase (hstat.len); /* Remember that we downloaded the file for later ".orig" code. */ if (*dt & ADDED_HTML_EXTENSION) @@ -1231,7 +1618,7 @@ The sizes do not match (local %ld) -- retrieving.\n"), local_size); else downloaded_file(FILE_DOWNLOADED_NORMALLY, locf); - free(filename_plus_orig_suffix); /* must precede every return! */ + xfree(filename_plus_orig_suffix); /* must precede every return! */ return RETROK; } else if (hstat.res == 0) /* No read error */ @@ -1249,7 +1636,7 @@ The sizes do not match (local %ld) -- retrieving.\n"), local_size); tms, u->url, hstat.len, locf, count); } ++opt.numurls; - opt.downloaded += hstat.len; + downloaded_increase (hstat.len); /* Remember that we downloaded the file for later ".orig" code. */ if (*dt & ADDED_HTML_EXTENSION) @@ -1257,7 +1644,7 @@ The sizes do not match (local %ld) -- retrieving.\n"), local_size); else downloaded_file(FILE_DOWNLOADED_NORMALLY, locf); - free(filename_plus_orig_suffix); /* must precede every return! */ + xfree (filename_plus_orig_suffix); /* must precede every return! */ return RETROK; } else if (hstat.len < hstat.contlen) /* meaning we lost the @@ -1278,7 +1665,7 @@ The sizes do not match (local %ld) -- retrieving.\n"), local_size); "%s URL:%s [%ld/%ld] -> \"%s\" [%d]\n", tms, u->url, hstat.len, hstat.contlen, locf, count); ++opt.numurls; - opt.downloaded += hstat.len; + downloaded_increase (hstat.len); /* Remember that we downloaded the file for later ".orig" code. */ if (*dt & ADDED_HTML_EXTENSION) @@ -1286,7 +1673,7 @@ The sizes do not match (local %ld) -- retrieving.\n"), local_size); else downloaded_file(FILE_DOWNLOADED_NORMALLY, locf); - free(filename_plus_orig_suffix); /* must precede every return! */ + xfree (filename_plus_orig_suffix); /* must precede every return! */ return RETROK; } else /* the same, but not accepted */ @@ -1322,7 +1709,7 @@ The sizes do not match (local %ld) -- retrieving.\n"), local_size); break; } while (!opt.ntry || (count < opt.ntry)); - free(filename_plus_orig_suffix); /* must precede every return! */ + xfree (filename_plus_orig_suffix); /* must precede every return! */ return TRYLIMEXC; } @@ -1352,7 +1739,7 @@ mktime_from_utc (struct tm *t) "^ *(GMT|[+-][0-9]|$)", 0 otherwise. P being NULL (a valid result of strptime()) is considered a failure and 0 is returned. */ static int -check_end (char *p) +check_end (const char *p) { if (!p) return 0; @@ -1360,7 +1747,7 @@ check_end (char *p) ++p; if (!*p || (p[0] == 'G' && p[1] == 'M' && p[2] == 'T') - || ((p[0] == '+' || p[1] == '-') && ISDIGIT (p[1]))) + || ((p[0] == '+' || p[0] == '-') && ISDIGIT (p[1]))) return 1; else return 0; @@ -1499,7 +1886,7 @@ basic_authentication_encode (const char *user, const char *passwd, sprintf (t1, "%s:%s", user, passwd); t2 = (char *)alloca (1 + len2); base64_encode (t1, t2, len1); - res = (char *)malloc (len2 + 11 + strlen (header)); + res = (char *)xmalloc (len2 + 11 + strlen (header)); sprintf (res, "%s: Basic %s\r\n", header, t2); return res; @@ -1700,7 +2087,7 @@ username=\"%s\", realm=\"%s\", nonce=\"%s\", uri=\"%s\", response=\"%s\"", #endif /* USE_DIGEST */ -#define HACK_O_MATIC(line, string_constant) \ +#define BEGINS_WITH(line, string_constant) \ (!strncasecmp (line, string_constant, sizeof (string_constant) - 1) \ && (ISSPACE (line[sizeof (string_constant) - 1]) \ || !line[sizeof (string_constant) - 1])) @@ -1708,10 +2095,12 @@ username=\"%s\", realm=\"%s\", nonce=\"%s\", uri=\"%s\", response=\"%s\"", static int known_authentication_scheme_p (const char *au) { - return HACK_O_MATIC (au, "Basic") || HACK_O_MATIC (au, "Digest"); + return BEGINS_WITH (au, "Basic") + || BEGINS_WITH (au, "Digest") + || BEGINS_WITH (au, "NTLM"); } -#undef HACK_O_MATIC +#undef BEGINS_WITH /* Create the HTTP authorization request header. When the `WWW-Authenticate' response header is seen, according to the @@ -1727,6 +2116,8 @@ create_authorization_line (const char *au, const char *user, if (!strncasecmp (au, "Basic", 5)) wwwauth = basic_authentication_encode (user, passwd, "Authorization"); + if (!strncasecmp (au, "NTLM", 4)) + wwwauth = basic_authentication_encode (user, passwd, "Authorization"); #ifdef USE_DIGEST else if (!strncasecmp (au, "Digest", 6)) wwwauth = digest_authentication_encode (au, user, passwd, method, path);