X-Git-Url: http://sjero.net/git/?p=wget;a=blobdiff_plain;f=src%2Fhttp.c;h=de1cb084dbb1d15baabb8266332d5558d54c06cc;hp=14176aac0af659d58f78e354db861f2c82d1ffbe;hb=d9fea91a0a319e348adb504bd3edff148ff3d8a0;hpb=30ac043b0a4a9a983dd1b50ce1c89ed953019292 diff --git a/src/http.c b/src/http.c index 14176aac..de1cb084 100644 --- a/src/http.c +++ b/src/http.c @@ -7,7 +7,7 @@ This file is part of GNU Wget. GNU Wget is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or -(at your option) any later version. + (at your option) any later version. GNU Wget is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of @@ -53,16 +53,17 @@ so, delete this exception statement from your version. */ # include # endif #endif +#ifndef errno +extern int errno; +#endif #include "wget.h" #include "utils.h" #include "url.h" #include "host.h" -#include "rbuf.h" #include "retr.h" #include "headers.h" #include "connect.h" -#include "fnmatch.h" #include "netrc.h" #ifdef HAVE_SSL # include "gen_sslfunc.h" @@ -71,24 +72,25 @@ so, delete this exception statement from your version. */ #ifdef USE_DIGEST # include "gen-md5.h" #endif +#include "convert.h" extern char *version_string; +extern LARGE_INT total_downloaded_bytes; -#ifndef errno -extern int errno; -#endif static int cookies_loaded_p; struct cookie_jar *wget_cookie_jar; #define TEXTHTML_S "text/html" +#define TEXTXHTML_S "application/xhtml+xml" #define HTTP_ACCEPT "*/*" /* Some status code validation macros: */ #define H_20X(x) (((x) >= 200) && ((x) < 300)) #define H_PARTIAL(x) ((x) == HTTP_STATUS_PARTIAL_CONTENTS) -#define H_REDIRECTED(x) (((x) == HTTP_STATUS_MOVED_PERMANENTLY) \ - || ((x) == HTTP_STATUS_MOVED_TEMPORARILY)) +#define H_REDIRECTED(x) ((x) == HTTP_STATUS_MOVED_PERMANENTLY \ + || (x) == HTTP_STATUS_MOVED_TEMPORARILY \ + || (x) == HTTP_STATUS_TEMPORARY_REDIRECT) /* HTTP/1.0 status codes from RFC1945, provided for reference. */ /* Successful 2xx. */ @@ -103,6 +105,7 @@ struct cookie_jar *wget_cookie_jar; #define HTTP_STATUS_MOVED_PERMANENTLY 301 #define HTTP_STATUS_MOVED_TEMPORARILY 302 #define HTTP_STATUS_NOT_MODIFIED 304 +#define HTTP_STATUS_TEMPORARY_REDIRECT 307 /* Client error 4xx. */ #define HTTP_STATUS_BAD_REQUEST 400 @@ -185,39 +188,29 @@ parse_http_status_line (const char *line, const char **reason_phrase_ptr) /* Send the contents of FILE_NAME to SOCK/SSL. Make sure that exactly PROMISED_SIZE bytes are sent over the wire -- if the file is - longer, read only that much; if the file is shorter, pad it with - zeros. */ + longer, read only that much; if the file is shorter, report an error. */ static int -post_file (int sock, void *ssl, const char *file_name, long promised_size) +post_file (int sock, const char *file_name, long promised_size) { static char chunk[8192]; long written = 0; int write_error; FILE *fp; - /* Only one of SOCK and SSL may be active at the same time. */ - assert (sock > -1 || ssl != NULL); - assert (sock == -1 || ssl == NULL); - DEBUGP (("[writing POST file %s ... ", file_name)); fp = fopen (file_name, "rb"); if (!fp) - goto pad; - while (written < promised_size) + return -1; + while (!feof (fp) && written < promised_size) { int towrite; int length = fread (chunk, 1, sizeof (chunk), fp); if (length == 0) break; towrite = WMIN (promised_size - written, length); -#ifdef HAVE_SSL - if (ssl) - write_error = ssl_iwrite (ssl, chunk, towrite); - else -#endif - write_error = iwrite (sock, chunk, towrite); + write_error = fd_write (sock, chunk, towrite, -1); if (write_error < 0) { fclose (fp); @@ -227,33 +220,36 @@ post_file (int sock, void *ssl, const char *file_name, long promised_size) } fclose (fp); - pad: + /* If we've written less than was promised, report a (probably + nonsensical) error rather than break the promise. */ if (written < promised_size) { - /* This highly unlikely case can happen only if the file has - shrunk under us. To uphold the promise that exactly - promised_size bytes would be delivered, pad the remaining - data with zeros. #### Should we abort instead? */ - DEBUGP (("padding %ld bytes ... ", promised_size - written)); - memset (chunk, '\0', sizeof (chunk)); - while (written < promised_size) - { - int towrite = WMIN (promised_size - written, sizeof (chunk)); -#ifdef HAVE_SSL - if (ssl) - write_error = ssl_iwrite (ssl, chunk, towrite); - else -#endif - write_error = iwrite (sock, chunk, towrite); - if (write_error < 0) - return -1; - written += towrite; - } + errno = EINVAL; + return -1; } + assert (written == promised_size); DEBUGP (("done]\n")); return 0; } + +static const char * +next_header (const char *h) +{ + const char *end = NULL; + const char *p = h; + do + { + p = strchr (p, '\n'); + if (!p) + return end; + end = ++p; + } + while (*p == ' ' || *p == '\t'); + + return end; +} + /* Functions to be used as arguments to header_process(): */ @@ -356,8 +352,7 @@ http_process_set_cookie (const char *hdr, void *arg) /* The jar should have been created by now. */ assert (wget_cookie_jar != NULL); - cookie_jar_process_set_cookie (wget_cookie_jar, u->host, u->port, u->path, - hdr); + cookie_handle_set_cookie (wget_cookie_jar, u->host, u->port, u->path, hdr); return 1; } @@ -365,45 +360,36 @@ http_process_set_cookie (const char *hdr, void *arg) /* Persistent connections. Currently, we cache the most recently used connection as persistent, provided that the HTTP server agrees to make it such. The persistence data is stored in the variables - below. Ideally, it would be in a structure, and it should be - possible to cache an arbitrary fixed number of these connections. - - I think the code is quite easy to extend in that direction. */ + below. Ideally, it should be possible to cache an arbitrary fixed + number of these connections. */ /* Whether a persistent connection is active. */ -static int pc_active_p; -/* Host and port of currently active persistent connection. */ -static struct address_list *pc_last_host_ip; -static unsigned short pc_last_port; +static int pconn_active; -/* File descriptor of the currently active persistent connection. */ -static int pc_last_fd; +static struct { + /* The socket of the connection. */ + int socket; -#ifdef HAVE_SSL -/* Whether a ssl handshake has occoured on this connection */ -static int pc_active_ssl; -/* SSL connection of the currently active persistent connection. */ -static SSL *pc_last_ssl; -#endif /* HAVE_SSL */ + /* Host and port of the currently active persistent connection. */ + char *host; + int port; + + /* Whether a ssl handshake has occoured on this connection. */ + int ssl; +} pconn; -/* Mark the persistent connection as invalid. This is used by the - CLOSE_* macros after they forcefully close a registered persistent - connection. This does not close the file descriptor -- it is left - to the caller to do that. (Maybe it should, though.) */ +/* Mark the persistent connection as invalid and free the resources it + uses. This is used by the CLOSE_* macros after they forcefully + close a registered persistent connection. */ static void invalidate_persistent (void) { - pc_active_p = 0; -#ifdef HAVE_SSL - pc_active_ssl = 0; -#endif /* HAVE_SSL */ - if (pc_last_host_ip != NULL) - { - address_list_release (pc_last_host_ip); - pc_last_host_ip = NULL; - } - DEBUGP (("Invalidating fd %d from further reuse.\n", pc_last_fd)); + DEBUGP (("Disabling further reuse of socket %d.\n", pconn.socket)); + pconn_active = 0; + fd_close (pconn.socket); + xfree (pconn.host); + xzero (pconn); } /* Register FD, which should be a TCP/IP connection to HOST:PORT, as @@ -414,131 +400,124 @@ invalidate_persistent (void) If a previous connection was persistent, it is closed. */ -#ifdef HAVE_SSL -static void -register_persistent (const char *host, unsigned short port, int fd, SSL *ssl) -{ -#else static void -register_persistent (const char *host, unsigned short port, int fd) +register_persistent (const char *host, int port, int fd, int ssl) { -#endif - if (pc_active_p) + if (pconn_active) { - if (pc_last_fd == fd) + if (pconn.socket == fd) { - /* The connection FD is already registered. Nothing to - do. */ + /* The connection FD is already registered. */ return; } else { - /* The old persistent connection is still active; let's - close it first. This situation arises whenever a - persistent connection exists, but we then connect to a - different host, and try to register a persistent - connection to that one. */ -#ifdef HAVE_SSL - /* The ssl disconnect has to take place before the closing - of pc_last_fd. */ - if (pc_last_ssl) - shutdown_ssl(pc_last_ssl); -#endif - CLOSE (pc_last_fd); + /* The old persistent connection is still active; close it + first. This situation arises whenever a persistent + connection exists, but we then connect to a different + host, and try to register a persistent connection to that + one. */ invalidate_persistent (); } } - assert (pc_last_host_ip == NULL); + pconn_active = 1; + pconn.socket = fd; + pconn.host = xstrdup (host); + pconn.port = port; + pconn.ssl = ssl; - /* This lookup_host cannot fail, because it has the results in the - cache. */ - pc_last_host_ip = lookup_host (host, 1); - assert (pc_last_host_ip != NULL); - - pc_last_port = port; - pc_last_fd = fd; - pc_active_p = 1; -#ifdef HAVE_SSL - pc_last_ssl = ssl; - pc_active_ssl = ssl ? 1 : 0; -#endif - DEBUGP (("Registered fd %d for persistent reuse.\n", fd)); + DEBUGP (("Registered socket %d for persistent reuse.\n", fd)); } -#ifdef HAVE_SSL -# define SHUTDOWN_SSL(ssl) do { \ - if (ssl) \ - shutdown_ssl (ssl); \ -} while (0) -#else -# define SHUTDOWN_SSL(ssl) -#endif - /* Return non-zero if a persistent connection is available for connecting to HOST:PORT. */ -#ifdef HAVE_SSL static int -persistent_available_p (const char *host, unsigned short port, int ssl) +persistent_available_p (const char *host, int port, int ssl, + int *host_lookup_failed) { -#else -static int -persistent_available_p (const char *host, unsigned short port) -{ -#endif - int success; - struct address_list *this_host_ip; - /* First, check whether a persistent connection is active at all. */ - if (!pc_active_p) - return 0; - /* Second, check if the active connection pertains to the correct - (HOST, PORT) ordered pair. */ - if (port != pc_last_port) + if (!pconn_active) return 0; -#ifdef HAVE_SSL - /* Second, a): check if current connection is (not) ssl, too. This - test is unlikely to fail because HTTP and HTTPS typicaly use - different ports. Yet it is possible, or so I [Christian - Fraenkel] have been told, to run HTTPS and HTTP simultaneus on - the same port. */ - if (ssl != pc_active_ssl) + /* If we want SSL and the last connection wasn't or vice versa, + don't use it. Checking for host and port is not enough because + HTTP and HTTPS can apparently coexist on the same port. */ + if (ssl != pconn.ssl) return 0; -#endif /* HAVE_SSL */ - this_host_ip = lookup_host (host, 1); - if (!this_host_ip) + /* If we're not connecting to the same port, we're not interested. */ + if (port != pconn.port) return 0; - /* To equate the two host names for the purposes of persistent - connections, they need to share all the IP addresses in the - list. */ - success = address_list_match_all (pc_last_host_ip, this_host_ip); - address_list_release (this_host_ip); - if (!success) - return 0; + /* If the host is the same, we're in business. If not, there is + still hope -- read below. */ + if (0 != strcasecmp (host, pconn.host)) + { + /* If pconn.socket is already talking to HOST, we needn't + reconnect. This happens often when both sites are virtual + hosts distinguished only by name and served by the same + network interface, and hence the same web server (possibly + set up by the ISP and serving many different web sites). + This admittedly non-standard optimization does not contradict + HTTP and works well with popular server software. */ + + int found; + ip_address ip; + struct address_list *al; + + if (ssl) + /* Don't try to talk to two different SSL sites over the same + secure connection! (Besides, it's not clear if name-based + virtual hosting is even possible with SSL.) */ + return 0; - /* Third: check whether the connection is still open. This is + /* If pconn.socket's peer is one of the IP addresses HOST + resolves to, pconn.socket is for all intents and purposes + already talking to HOST. */ + + if (!socket_ip_address (pconn.socket, &ip, ENDPOINT_PEER)) + { + /* Can't get the peer's address -- something must be very + wrong with the connection. */ + invalidate_persistent (); + return 0; + } + al = lookup_host (host, 0); + if (!al) + { + *host_lookup_failed = 1; + return 0; + } + + found = address_list_contains (al, &ip); + address_list_release (al); + + if (!found) + return 0; + + /* The persistent connection's peer address was found among the + addresses HOST resolved to; therefore, pconn.sock is in fact + already talking to HOST -- no need to reconnect. */ + } + + /* Finally, check whether the connection is still open. This is important because most server implement a liberal (short) timeout on persistent connections. Wget can of course always reconnect if the connection doesn't work out, but it's nicer to know in advance. This test is a logical followup of the first test, but is "expensive" and therefore placed at the end of the list. */ - if (!test_socket_open (pc_last_fd)) + + if (!test_socket_open (pconn.socket)) { /* Oops, the socket is no longer open. Now that we know that, let's invalidate the persistent connection before returning 0. */ - CLOSE (pc_last_fd); -#ifdef HAVE_SSL - SHUTDOWN_SSL (pc_last_ssl); - pc_last_ssl = NULL; -#endif invalidate_persistent (); return 0; } + return 1; } @@ -560,18 +539,18 @@ persistent_available_p (const char *host, unsigned short port) #define CLOSE_FINISH(fd) do { \ if (!keep_alive) \ { \ - SHUTDOWN_SSL (ssl); \ - CLOSE (fd); \ - if (pc_active_p && (fd) == pc_last_fd) \ + if (pconn_active && (fd) == pconn.socket) \ invalidate_persistent (); \ + else \ + fd_close (fd); \ } \ } while (0) #define CLOSE_INVALIDATE(fd) do { \ - SHUTDOWN_SSL (ssl); \ - CLOSE (fd); \ - if (pc_active_p && (fd) == pc_last_fd) \ + if (pconn_active && (fd) == pconn.socket) \ invalidate_persistent (); \ + else \ + fd_close (fd); \ } while (0) struct http_stat @@ -584,7 +563,7 @@ struct http_stat char *remote_time; /* remote time-stamp string */ char *error; /* textual HTTP error */ int statcode; /* status code */ - long dltime; /* time of the download */ + double dltime; /* time of the download in msecs */ int no_truncate; /* whether truncating the file is forbidden. */ const char *referer; /* value of the referer header. */ @@ -594,9 +573,9 @@ struct http_stat static void free_hstat (struct http_stat *hs) { - FREE_MAYBE (hs->newloc); - FREE_MAYBE (hs->remote_time); - FREE_MAYBE (hs->error); + xfree_null (hs->newloc); + xfree_null (hs->remote_time); + xfree_null (hs->error); /* Guard against being called twice. */ hs->newloc = NULL; @@ -624,9 +603,7 @@ time_t http_atotm PARAMS ((const char *)); will print it if there is enough information to do so (almost always), returning the error to the caller (i.e. http_loop). - Various HTTP parameters are stored to hs. Although it parses the - response code correctly, it is not used in a sane way. The caller - can do that, though. + Various HTTP parameters are stored to hs. If PROXY is non-NULL, the connection will be made to the proxy server, and u->url will be requested. */ @@ -638,22 +615,20 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy) char *pragma_h, *referer, *useragent, *range, *wwwauth; char *authenticate_h; char *proxyauth; - char *all_headers; char *port_maybe; char *request_keep_alive; - int sock, hcount, all_length, statcode; + int sock, hcount, statcode; int write_error; long contlen, contrange; struct url *conn; FILE *fp; int auth_tried_already; - struct rbuf rbuf; -#ifdef HAVE_SSL - static SSL_CTX *ssl_ctx = NULL; - SSL *ssl = NULL; -#endif + int using_ssl = 0; char *cookies = NULL; + char *head; + const char *hdr_beg, *hdr_end; + /* Whether this connection will be kept alive after the HTTP request is done. */ int keep_alive; @@ -674,41 +649,34 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy) char *post_content_type, *post_content_length; long post_data_size = 0; + int host_lookup_failed; + #ifdef HAVE_SSL - /* initialize ssl_ctx on first run */ - if (!ssl_ctx) + /* Initialize the SSL context. After the first run, this is a + no-op. */ + switch (ssl_init ()) { - uerr_t err = init_ssl (&ssl_ctx); - if (err != 0) - { - switch (err) - { - case SSLERRCTXCREATE: - /* this is fatal */ - logprintf (LOG_NOTQUIET, _("Failed to set up an SSL context\n")); - ssl_printerrors (); - return err; - case SSLERRCERTFILE: - /* try without certfile */ - logprintf (LOG_NOTQUIET, - _("Failed to load certificates from %s\n"), - opt.sslcertfile); - ssl_printerrors (); - logprintf (LOG_NOTQUIET, - _("Trying without the specified certificate\n")); - break; - case SSLERRCERTKEY: - logprintf (LOG_NOTQUIET, - _("Failed to get certificate key from %s\n"), - opt.sslcertkey); - ssl_printerrors (); - logprintf (LOG_NOTQUIET, - _("Trying without the specified certificate\n")); - break; - default: - break; - } - } + case SSLERRCTXCREATE: + /* this is fatal */ + logprintf (LOG_NOTQUIET, _("Failed to set up an SSL context\n")); + return SSLERRCTXCREATE; + case SSLERRCERTFILE: + /* try without certfile */ + logprintf (LOG_NOTQUIET, + _("Failed to load certificates from %s\n"), + opt.sslcertfile); + logprintf (LOG_NOTQUIET, + _("Trying without the specified certificate\n")); + break; + case SSLERRCERTKEY: + logprintf (LOG_NOTQUIET, + _("Failed to get certificate key from %s\n"), + opt.sslcertkey); + logprintf (LOG_NOTQUIET, + _("Trying without the specified certificate\n")); + break; + default: + break; } #endif /* HAVE_SSL */ @@ -745,50 +713,52 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy) server. */ conn = proxy ? proxy : u; + host_lookup_failed = 0; + /* First: establish the connection. */ if (inhibit_keep_alive - || -#ifndef HAVE_SSL - !persistent_available_p (conn->host, conn->port) + || !persistent_available_p (conn->host, conn->port, +#ifdef HAVE_SSL + u->scheme == SCHEME_HTTPS #else - !persistent_available_p (conn->host, conn->port, - u->scheme == SCHEME_HTTPS) -#endif /* HAVE_SSL */ - ) + 0 +#endif + , &host_lookup_failed)) { - struct address_list *al = lookup_host (conn->host, 0); - if (!al) + /* In its current implementation, persistent_available_p will + look up conn->host in some cases. If that lookup failed, we + don't need to bother with connect_to_host. */ + if (host_lookup_failed) return HOSTERR; - set_connection_host_name (conn->host); - sock = connect_to_many (al, conn->port, 0); - set_connection_host_name (NULL); - address_list_release (al); - if (sock < 0) - return CONNECT_ERROR (errno); + sock = connect_to_host (conn->host, conn->port); + if (sock == E_HOST) + return HOSTERR; + else if (sock < 0) + return (retryable_socket_connect_error (errno) + ? CONERROR : CONIMPOSSIBLE); #ifdef HAVE_SSL if (conn->scheme == SCHEME_HTTPS) - if (connect_ssl (&ssl, ssl_ctx,sock) != 0) - { - logputs (LOG_VERBOSE, "\n"); - logprintf (LOG_NOTQUIET, _("Unable to establish SSL connection.\n")); - ssl_printerrors (); - CLOSE (sock); - return CONSSLERR; - } + { + if (!ssl_connect (sock)) + { + logputs (LOG_VERBOSE, "\n"); + logprintf (LOG_NOTQUIET, + _("Unable to establish SSL connection.\n")); + fd_close (sock); + return CONSSLERR; + } + using_ssl = 1; + } #endif /* HAVE_SSL */ } else { - logprintf (LOG_VERBOSE, _("Reusing connection to %s:%hu.\n"), - conn->host, conn->port); - /* #### pc_last_fd should be accessed through an accessor - function. */ - sock = pc_last_fd; -#ifdef HAVE_SSL - ssl = pc_last_ssl; -#endif /* HAVE_SSL */ + logprintf (LOG_VERBOSE, _("Reusing existing connection to %s:%d.\n"), + pconn.host, pconn.port); + sock = pconn.socket; + using_ssl = pconn.ssl; DEBUGP (("Reusing fd %d.\n", sock)); } @@ -918,14 +888,13 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy) request_keep_alive = NULL; if (opt.cookies) - cookies = cookie_jar_generate_cookie_header (wget_cookie_jar, u->host, - u->port, u->path, + cookies = cookie_header (wget_cookie_jar, u->host, u->port, u->path, #ifdef HAVE_SSL - u->scheme == SCHEME_HTTPS + u->scheme == SCHEME_HTTPS #else - 0 + 0 #endif - ); + ); if (opt.post_data || opt.post_file_name) { @@ -1001,45 +970,26 @@ Accept: %s\r\n\ post_content_type ? post_content_type : "", post_content_length ? post_content_length : "", opt.user_header ? opt.user_header : ""); - DEBUGP (("---request begin---\n%s", request)); + DEBUGP (("\n---request begin---\n%s", request)); /* Free the temporary memory. */ - FREE_MAYBE (wwwauth); - FREE_MAYBE (proxyauth); - FREE_MAYBE (cookies); + xfree_null (wwwauth); + xfree_null (proxyauth); + xfree_null (cookies); xfree (full_path); /* Send the request to server. */ -#ifdef HAVE_SSL - if (conn->scheme == SCHEME_HTTPS) - write_error = ssl_iwrite (ssl, request, strlen (request)); - else -#endif - write_error = iwrite (sock, request, strlen (request)); + write_error = fd_write (sock, request, strlen (request), -1); if (write_error >= 0) { if (opt.post_data) { DEBUGP (("[POST data: %s]\n", opt.post_data)); -#ifdef HAVE_SSL - if (conn->scheme == SCHEME_HTTPS) - write_error = ssl_iwrite (ssl, opt.post_data, post_data_size); - else -#endif - write_error = iwrite (sock, opt.post_data, post_data_size); - } - else if (opt.post_file_name) - { -#ifdef HAVE_SSL - if (conn->scheme == SCHEME_HTTPS) - write_error = post_file (-1, ssl, opt.post_file_name, - post_data_size); - else -#endif - write_error = post_file (sock, NULL, opt.post_file_name, - post_data_size); + write_error = fd_write (sock, opt.post_data, post_data_size, -1); } + else if (opt.post_file_name && post_data_size != 0) + write_error = post_file (sock, opt.post_file_name, post_data_size); } DEBUGP (("---request end---\n")); @@ -1057,73 +1007,43 @@ Accept: %s\r\n\ statcode = -1; *dt &= ~RETROKF; - /* Before reading anything, initialize the rbuf. */ - rbuf_initialize (&rbuf, sock); -#ifdef HAVE_SSL - if (conn->scheme == SCHEME_HTTPS) - rbuf.ssl = ssl; - else - rbuf.ssl = NULL; -#endif /* HAVE_SSL */ - all_headers = NULL; - all_length = 0; - /* Header-fetching loop. */ - hcount = 0; - while (1) - { - char *hdr; - int status; - - ++hcount; - /* Get the header. */ - status = header_get (&rbuf, &hdr, - /* Disallow continuations for status line. */ - (hcount == 1 ? HG_NO_CONTINUATIONS : HG_NONE)); + DEBUGP (("\n---response begin---\n")); - /* Check for errors. */ - if (status == HG_EOF && *hdr) + head = fd_read_head (sock); + if (!head) + { + logputs (LOG_VERBOSE, "\n"); + if (errno == 0) { - /* This used to be an unconditional error, but that was - somewhat controversial, because of a large number of - broken CGI's that happily "forget" to send the second EOL - before closing the connection of a HEAD request. - - So, the deal is to check whether the header is empty - (*hdr is zero if it is); if yes, it means that the - previous header was fully retrieved, and that -- most - probably -- the request is complete. "...be liberal in - what you accept." Oh boy. */ - logputs (LOG_VERBOSE, "\n"); - logputs (LOG_NOTQUIET, _("End of file while parsing headers.\n")); - xfree (hdr); - FREE_MAYBE (type); - FREE_MAYBE (all_headers); + logputs (LOG_NOTQUIET, _("No data received.\n")); CLOSE_INVALIDATE (sock); return HEOF; } - else if (status == HG_ERROR) + else { - logputs (LOG_VERBOSE, "\n"); logprintf (LOG_NOTQUIET, _("Read error (%s) in headers.\n"), strerror (errno)); - xfree (hdr); - FREE_MAYBE (type); - FREE_MAYBE (all_headers); CLOSE_INVALIDATE (sock); return HERR; } + } - /* If the headers are to be saved to a file later, save them to - memory now. */ - if (opt.save_headers) - { - int lh = strlen (hdr); - all_headers = (char *)xrealloc (all_headers, all_length + lh + 2); - memcpy (all_headers + all_length, hdr, lh); - all_length += lh; - all_headers[all_length++] = '\n'; - all_headers[all_length] = '\0'; - } + /* Loop through the headers and process them. */ + + hcount = 0; + for (hdr_beg = head; + (hdr_end = next_header (hdr_beg)); + hdr_beg = hdr_end) + { + char *hdr = strdupdelim (hdr_beg, hdr_end); + { + char *tmp = hdr + strlen (hdr); + if (tmp > hdr && tmp[-1] == '\n') + *--tmp = '\0'; + if (tmp > hdr && tmp[-1] == '\r') + *--tmp = '\0'; + } + ++hcount; /* Check for status line. */ if (hcount == 1) @@ -1151,7 +1071,7 @@ Accept: %s\r\n\ hs->error = xstrdup (error); if ((statcode != -1) -#ifdef DEBUG +#ifdef ENABLE_DEBUG && !opt.debug #endif ) @@ -1246,6 +1166,7 @@ Accept: %s\r\n\ done_header: xfree (hdr); } + DEBUGP (("---response end---\n")); logputs (LOG_VERBOSE, "\n"); @@ -1258,17 +1179,13 @@ Accept: %s\r\n\ if (keep_alive) /* The server has promised that it will not close the connection when we're done. This means that we can register it. */ -#ifndef HAVE_SSL - register_persistent (conn->host, conn->port, sock); -#else - register_persistent (conn->host, conn->port, sock, ssl); -#endif /* HAVE_SSL */ + register_persistent (conn->host, conn->port, sock, using_ssl); if ((statcode == HTTP_STATUS_UNAUTHORIZED) && authenticate_h) { /* Authorization is required. */ - FREE_MAYBE (type); + xfree_null (type); type = NULL; free_hstat (hs); CLOSE_INVALIDATE (sock); /* would be CLOSE_FINISH, but there @@ -1330,8 +1247,7 @@ Accept: %s\r\n\ hs->newloc ? _(" [following]") : ""); CLOSE_INVALIDATE (sock); /* would be CLOSE_FINISH, but there might be more bytes in the body. */ - FREE_MAYBE (type); - FREE_MAYBE (all_headers); + xfree_null (type); return NEWLOCATION; } } @@ -1339,7 +1255,9 @@ Accept: %s\r\n\ /* If content-type is not given, assume text/html. This is because of the multitude of broken CGI's that "forget" to generate the content-type. */ - if (!type || 0 == strncasecmp (type, TEXTHTML_S, strlen (TEXTHTML_S))) + if (!type || + 0 == strncasecmp (type, TEXTHTML_S, strlen (TEXTHTML_S)) || + 0 == strncasecmp (type, TEXTXHTML_S, strlen (TEXTXHTML_S))) *dt |= TEXTHTML; else *dt &= ~TEXTHTML; @@ -1351,9 +1269,9 @@ Accept: %s\r\n\ { char* last_period_in_local_filename = strrchr(*hs->local_file, '.'); - if (last_period_in_local_filename == NULL || - !(strcasecmp(last_period_in_local_filename, ".htm") == EQ || - strcasecmp(last_period_in_local_filename, ".html") == EQ)) + if (last_period_in_local_filename == NULL + || !(0 == strcasecmp (last_period_in_local_filename, ".htm") + || 0 == strcasecmp (last_period_in_local_filename, ".html"))) { size_t local_filename_len = strlen(*hs->local_file); @@ -1399,8 +1317,7 @@ Accept: %s\r\n\ hs->res = 0; /* Mark as successfully retrieved. */ *dt |= RETROKF; - FREE_MAYBE (type); - FREE_MAYBE (all_headers); + xfree_null (type); CLOSE_INVALIDATE (sock); /* would be CLOSE_FINISH, but there might be more bytes in the body. */ return RETRUNNEEDED; @@ -1414,8 +1331,7 @@ Accept: %s\r\n\ \n\ Continued download failed on this file, which conflicts with `-c'.\n\ Refusing to truncate existing file `%s'.\n\n"), *hs->local_file); - FREE_MAYBE (type); - FREE_MAYBE (all_headers); + xfree_null (type); CLOSE_INVALIDATE (sock); return CONTNOTSUPPORTED; } @@ -1430,8 +1346,7 @@ Refusing to truncate existing file `%s'.\n\n"), *hs->local_file); { /* This means the whole request was somehow misunderstood by the server. Bail out. */ - FREE_MAYBE (type); - FREE_MAYBE (all_headers); + xfree_null (type); CLOSE_INVALIDATE (sock); return RANGEERR; } @@ -1470,7 +1385,7 @@ Refusing to truncate existing file `%s'.\n\n"), *hs->local_file); logputs (LOG_VERBOSE, "\n"); } } - FREE_MAYBE (type); + xfree_null (type); type = NULL; /* We don't need it any more. */ /* Return if we have no intention of further downloading. */ @@ -1479,8 +1394,7 @@ Refusing to truncate existing file `%s'.\n\n"), *hs->local_file); /* In case the caller cares to look... */ hs->len = 0L; hs->res = 0; - FREE_MAYBE (type); - FREE_MAYBE (all_headers); + xfree_null (type); CLOSE_INVALIDATE (sock); /* would be CLOSE_FINISH, but there might be more bytes in the body. */ return RETRFINISHED; @@ -1498,7 +1412,6 @@ Refusing to truncate existing file `%s'.\n\n"), *hs->local_file); logprintf (LOG_NOTQUIET, "%s: %s\n", *hs->local_file, strerror (errno)); CLOSE_INVALIDATE (sock); /* would be CLOSE_FINISH, but there might be more bytes in the body. */ - FREE_MAYBE (all_headers); return FOPENERR; } } @@ -1538,12 +1451,12 @@ Refusing to truncate existing file `%s'.\n\n"), *hs->local_file); /* #### This confuses the code that checks for file size. There should be some overhead information. */ if (opt.save_headers) - fwrite (all_headers, 1, all_length, fp); + fwrite (head, 1, strlen (head), fp); /* Get the contents of the document. */ - hs->res = get_contents (sock, fp, &hs->len, hs->restval, + hs->res = fd_read_body (sock, fp, &hs->len, hs->restval, (contlen != -1 ? contlen : 0), - &rbuf, keep_alive, &hs->dltime); + keep_alive, &hs->dltime); if (hs->res >= 0) CLOSE_FINISH (sock); @@ -1562,7 +1475,6 @@ Refusing to truncate existing file `%s'.\n\n"), *hs->local_file); if (flush_res == EOF) hs->res = -2; } - FREE_MAYBE (all_headers); if (hs->res == -2) return FWRITEERR; return RETRFINISHED; @@ -1614,12 +1526,12 @@ http_loop (struct url *u, char **newloc, char **local_file, const char *referer, hstat.local_file = local_file; else if (local_file) { - *local_file = url_filename (u); + *local_file = url_file_name (u); hstat.local_file = local_file; } else { - dummy = url_filename (u); + dummy = url_file_name (u); hstat.local_file = &dummy; } @@ -1647,14 +1559,14 @@ File `%s' already there, will not retrieve.\n"), *hstat.local_file); if (has_html_suffix_p (*hstat.local_file)) *dt |= TEXTHTML; - FREE_MAYBE (dummy); + xfree_null (dummy); return RETROK; } use_ts = 0; if (opt.timestamping) { - boolean local_dot_orig_file_exists = FALSE; + int local_dot_orig_file_exists = 0; if (opt.backup_converted) /* If -K is specified, we'll act on the assumption that it was specified @@ -1681,7 +1593,7 @@ File `%s' already there, will not retrieve.\n"), *hstat.local_file); /* Try to stat() the .orig file. */ if (stat (filename_plus_orig_suffix, &st) == 0) { - local_dot_orig_file_exists = TRUE; + local_dot_orig_file_exists = 1; local_filename = filename_plus_orig_suffix; } } @@ -1799,11 +1711,11 @@ File `%s' already there, will not retrieve.\n"), *hstat.local_file); printwhat (count, opt.ntry); continue; break; - case HOSTERR: case CONREFUSED: case PROXERR: case AUTHFAILED: + case HOSTERR: case CONIMPOSSIBLE: case PROXERR: case AUTHFAILED: case SSLERRCTXCREATE: case CONTNOTSUPPORTED: /* Fatal errors just return from the function. */ free_hstat (&hstat); - FREE_MAYBE (dummy); + xfree_null (dummy); return err; break; case FWRITEERR: case FOPENERR: @@ -1812,7 +1724,7 @@ File `%s' already there, will not retrieve.\n"), *hstat.local_file); logprintf (LOG_NOTQUIET, _("Cannot write to `%s' (%s).\n"), *hstat.local_file, strerror (errno)); free_hstat (&hstat); - FREE_MAYBE (dummy); + xfree_null (dummy); return err; break; case CONSSLERR: @@ -1820,7 +1732,7 @@ File `%s' already there, will not retrieve.\n"), *hstat.local_file); logputs (LOG_VERBOSE, "\n"); logprintf (LOG_NOTQUIET, _("Unable to establish SSL connection.\n")); free_hstat (&hstat); - FREE_MAYBE (dummy); + xfree_null (dummy); return err; break; case NEWLOCATION: @@ -1831,17 +1743,17 @@ File `%s' already there, will not retrieve.\n"), *hstat.local_file); _("ERROR: Redirection (%d) without location.\n"), hstat.statcode); free_hstat (&hstat); - FREE_MAYBE (dummy); + xfree_null (dummy); return WRONGCODE; } free_hstat (&hstat); - FREE_MAYBE (dummy); + xfree_null (dummy); return NEWLOCATION; break; case RETRUNNEEDED: /* The file was already fully retrieved. */ free_hstat (&hstat); - FREE_MAYBE (dummy); + xfree_null (dummy); return RETROK; break; case RETRFINISHED: @@ -1864,7 +1776,7 @@ File `%s' already there, will not retrieve.\n"), *hstat.local_file); tms, hstat.statcode, hstat.error); logputs (LOG_VERBOSE, "\n"); free_hstat (&hstat); - FREE_MAYBE (dummy); + xfree_null (dummy); return WRONGCODE; } @@ -1908,7 +1820,7 @@ Last-modified header invalid -- time-stamp ignored.\n")); Server file no newer than local file `%s' -- not retrieving.\n\n"), local_filename); free_hstat (&hstat); - FREE_MAYBE (dummy); + xfree_null (dummy); return RETROK; } else if (tml >= tmr) @@ -1946,7 +1858,7 @@ The sizes do not match (local %ld) -- retrieving.\n"), local_size); if (opt.spider) { logprintf (LOG_NOTQUIET, "%d %s\n\n", hstat.statcode, hstat.error); - FREE_MAYBE (dummy); + xfree_null (dummy); return RETROK; } @@ -1964,7 +1876,7 @@ The sizes do not match (local %ld) -- retrieving.\n"), local_size); tms, u->url, hstat.len, hstat.contlen, locf, count); } ++opt.numurls; - downloaded_increase (hstat.len); + total_downloaded_bytes += hstat.len; /* Remember that we downloaded the file for later ".orig" code. */ if (*dt & ADDED_HTML_EXTENSION) @@ -1973,7 +1885,7 @@ The sizes do not match (local %ld) -- retrieving.\n"), local_size); downloaded_file(FILE_DOWNLOADED_NORMALLY, locf); free_hstat (&hstat); - FREE_MAYBE (dummy); + xfree_null (dummy); return RETROK; } else if (hstat.res == 0) /* No read error */ @@ -1991,7 +1903,7 @@ The sizes do not match (local %ld) -- retrieving.\n"), local_size); tms, u->url, hstat.len, locf, count); } ++opt.numurls; - downloaded_increase (hstat.len); + total_downloaded_bytes += hstat.len; /* Remember that we downloaded the file for later ".orig" code. */ if (*dt & ADDED_HTML_EXTENSION) @@ -2000,7 +1912,7 @@ The sizes do not match (local %ld) -- retrieving.\n"), local_size); downloaded_file(FILE_DOWNLOADED_NORMALLY, locf); free_hstat (&hstat); - FREE_MAYBE (dummy); + xfree_null (dummy); return RETROK; } else if (hstat.len < hstat.contlen) /* meaning we lost the @@ -2022,7 +1934,7 @@ The sizes do not match (local %ld) -- retrieving.\n"), local_size); "%s URL:%s [%ld/%ld] -> \"%s\" [%d]\n", tms, u->url, hstat.len, hstat.contlen, locf, count); ++opt.numurls; - downloaded_increase (hstat.len); + total_downloaded_bytes += hstat.len; /* Remember that we downloaded the file for later ".orig" code. */ if (*dt & ADDED_HTML_EXTENSION) @@ -2031,7 +1943,7 @@ The sizes do not match (local %ld) -- retrieving.\n"), local_size); downloaded_file(FILE_DOWNLOADED_NORMALLY, locf); free_hstat (&hstat); - FREE_MAYBE (dummy); + xfree_null (dummy); return RETROK; } else /* the same, but not accepted */ @@ -2227,7 +2139,7 @@ http_atotm (const char *time_string) GNU strptime does not have this problem because it recognizes both international and local dates. */ - for (i = 0; i < ARRAY_SIZE (time_formats); i++) + for (i = 0; i < countof (time_formats); i++) if (check_end (strptime (time_string, time_formats[i], &t))) return mktime_from_utc (&t); @@ -2338,7 +2250,7 @@ extract_header_attr (const char *au, const char *attr_name, char **ret) ; if (!*ep) return -1; - FREE_MAYBE (*ret); + xfree_null (*ret); *ret = strdupdelim (cp, ep); return ep - au + 1; } @@ -2357,8 +2269,8 @@ dump_hash (unsigned char *buf, const unsigned char *hash) for (i = 0; i < MD5_HASHLEN; i++, hash++) { - *buf++ = XDIGIT_TO_xchar (*hash >> 4); - *buf++ = XDIGIT_TO_xchar (*hash & 0xf); + *buf++ = XNUM_TO_digit (*hash >> 4); + *buf++ = XNUM_TO_digit (*hash & 0xf); } *buf = '\0'; } @@ -2389,15 +2301,15 @@ digest_authentication_encode (const char *au, const char *user, int i; au += skip_lws (au); - for (i = 0; i < ARRAY_SIZE (options); i++) + for (i = 0; i < countof (options); i++) { int skip = extract_header_attr (au, options[i].name, options[i].variable); if (skip < 0) { - FREE_MAYBE (realm); - FREE_MAYBE (opaque); - FREE_MAYBE (nonce); + xfree_null (realm); + xfree_null (opaque); + xfree_null (nonce); return NULL; } else if (skip) @@ -2406,7 +2318,7 @@ digest_authentication_encode (const char *au, const char *user, break; } } - if (i == ARRAY_SIZE (options)) + if (i == countof (options)) { while (*au && *au != '=') au++; @@ -2430,9 +2342,9 @@ digest_authentication_encode (const char *au, const char *user, } if (!realm || !nonce || !user || !passwd || !path || !method) { - FREE_MAYBE (realm); - FREE_MAYBE (opaque); - FREE_MAYBE (nonce); + xfree_null (realm); + xfree_null (opaque); + xfree_null (nonce); return NULL; } @@ -2537,6 +2449,4 @@ create_authorization_line (const char *au, const char *user, void http_cleanup (void) { - if (pc_last_host_ip) - address_list_release (pc_last_host_ip); }