/* HTTP support.
- Copyright (C) 2005 Free Software Foundation, Inc.
+ Copyright (C) 1996-2005 Free Software Foundation, Inc.
This file is part of GNU Wget.
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
-along with Wget; if not, write to the Free Software
-Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+along with Wget; if not, write to the Free Software Foundation, Inc.,
+51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
In addition, as a special exception, the Free Software Foundation
gives permission to link the code of its release of Wget with the
#include <locale.h>
#include "wget.h"
+#include "http.h"
#include "utils.h"
#include "url.h"
#include "host.h"
extern char *version_string;
-extern FILE *output_stream;
-extern bool output_stream_regular;
-
#ifndef MIN
# define MIN(x, y) ((x) > (y) ? (y) : (x))
#endif
write_error = fd_write (fd, request_string, size - 1, -1);
if (write_error < 0)
logprintf (LOG_VERBOSE, _("Failed writing HTTP request: %s.\n"),
- strerror (errno));
+ fd_errstr (fd));
return write_error;
}
return 0;
}
\f
+/* Determine whether [START, PEEKED + PEEKLEN) contains an empty line.
+ If so, return the pointer to the position after the line, otherwise
+ return NULL. This is used as callback to fd_read_hunk. The data
+ between START and PEEKED has been read and cannot be "unread"; the
+ data after PEEKED has only been peeked. */
+
static const char *
-response_head_terminator (const char *hunk, int oldlen, int peeklen)
+response_head_terminator (const char *start, const char *peeked, int peeklen)
{
- const char *start, *end;
+ const char *p, *end;
/* If at first peek, verify whether HUNK starts with "HTTP". If
not, this is a HTTP/0.9 request and we must bail out without
reading anything. */
- if (oldlen == 0 && 0 != memcmp (hunk, "HTTP", MIN (peeklen, 4)))
- return hunk;
-
- if (oldlen < 4)
- start = hunk;
- else
- start = hunk + oldlen - 4;
- end = hunk + oldlen + peeklen;
-
- for (; start < end - 1; start++)
- if (*start == '\n')
+ if (start == peeked && 0 != memcmp (start, "HTTP", MIN (peeklen, 4)))
+ return start;
+
+ /* Look for "\n[\r]\n", and return the following position if found.
+ Start two chars before the current to cover the possibility that
+ part of the terminator (e.g. "\n\r") arrived in the previous
+ batch. */
+ p = peeked - start < 2 ? start : peeked - 2;
+ end = peeked + peeklen;
+
+ /* Check for \n\r\n or \n\n anywhere in [p, end-2). */
+ for (; p < end - 2; p++)
+ if (*p == '\n')
{
- if (start < end - 2
- && start[1] == '\r'
- && start[2] == '\n')
- return start + 3;
- if (start[1] == '\n')
- return start + 2;
+ if (p[1] == '\r' && p[2] == '\n')
+ return p + 3;
+ else if (p[1] == '\n')
+ return p + 2;
}
+ /* p==end-2: check for \n\n directly preceding END. */
+ if (p[0] == '\n' && p[1] == '\n')
+ return p + 2;
+
return NULL;
}
-/* The maximum size of a single HTTP response we care to read. This
- is not meant to impose an arbitrary limit, but to protect the user
- from Wget slurping up available memory upon encountering malicious
- or buggy server output. Define it to 0 to remove the limit. */
+/* The maximum size of a single HTTP response we care to read. Rather
+ than being a limit of the reader implementation, this limit
+ prevents Wget from slurping all available memory upon encountering
+ malicious or buggy server output, thus protecting the user. Define
+ it to 0 to remove the limit. */
#define HTTP_RESPONSE_MAX_SIZE 65536
/* Don't normally report the error since this is an
optimization that should be invisible to the user. */
DEBUGP (("] aborting (%s).\n",
- ret < 0 ? strerror (errno) : "EOF received"));
+ ret < 0 ? fd_errstr (fd) : "EOF received"));
return false;
}
contlen -= ret;
}
/* Finally, check whether the connection is still open. This is
- important because most server implement a liberal (short) timeout
+ important because most servers implement liberal (short) timeout
on persistent connections. Wget can of course always reconnect
if the connection doesn't work out, but it's nicer to know in
advance. This test is a logical followup of the first test, but
- is "expensive" and therefore placed at the end of the list. */
+ is "expensive" and therefore placed at the end of the list.
+
+ (Current implementation of test_socket_open has a nice side
+ effect that it treats sockets with pending data as "closed".
+ This is exactly what we want: if a broken server sends message
+ body in response to HEAD, or if it sends more than conent-length
+ data, we won't reuse the corrupted connection.) */
if (!test_socket_open (pconn.socket))
{
wgint contlen; /* expected length */
wgint restval; /* the restart value */
int res; /* the result of last read */
+ char *rderrmsg; /* error message from read error */
char *newloc; /* new location (redirection) */
char *remote_time; /* remote time-stamp string */
char *error; /* textual HTTP error */
xfree_null (hs->newloc);
xfree_null (hs->remote_time);
xfree_null (hs->error);
+ xfree_null (hs->rderrmsg);
/* Guard against being called twice. */
hs->newloc = NULL;
static char *basic_authentication_encode (const char *, const char *);
static bool known_authentication_scheme_p (const char *, const char *);
-time_t http_atotm (const char *);
-
#define BEGINS_WITH(line, string_constant) \
(!strncasecmp (line, string_constant, sizeof (string_constant) - 1) \
&& (ISSPACE (line[sizeof (string_constant) - 1]) \
hs->len = 0;
hs->contlen = -1;
hs->res = -1;
+ hs->rderrmsg = NULL;
hs->newloc = NULL;
hs->remote_time = NULL;
hs->error = NULL;
request_set_header (req, "Proxy-Authorization", proxyauth, rel_value);
}
+ /* Generate the Host header, HOST:PORT. Take into account that:
+
+ - Broken server-side software often doesn't recognize the PORT
+ argument, so we must generate "Host: www.server.com" instead of
+ "Host: www.server.com:80" (and likewise for https port).
+
+ - IPv6 addresses contain ":", so "Host: 3ffe:8100:200:2::2:1234"
+ becomes ambiguous and needs to be rewritten as "Host:
+ [3ffe:8100:200:2::2]:1234". */
{
- /* Whether we need to print the host header with braces around
- host, e.g. "Host: [3ffe:8100:200:2::2]:1234" instead of the
- usual "Host: symbolic-name:1234". */
- bool squares = strchr (u->host, ':') != NULL;
- if (u->port == scheme_default_port (u->scheme))
- request_set_header (req, "Host",
- aprintf (squares ? "[%s]" : "%s", u->host),
- rel_value);
- else
- request_set_header (req, "Host",
- aprintf (squares ? "[%s]:%d" : "%s:%d",
- u->host, u->port),
- rel_value);
+ /* Formats arranged for hfmt[add_port][add_squares]. */
+ static const char *hfmt[][2] = {
+ { "%s", "[%s]" }, { "%s:%d", "[%s]:%d" }
+ };
+ int add_port = u->port != scheme_default_port (u->scheme);
+ int add_squares = strchr (u->host, ':') != NULL;
+ request_set_header (req, "Host",
+ aprintf (hfmt[add_port][add_squares], u->host, u->port),
+ rel_value);
}
if (!inhibit_keep_alive)
request_free (connreq);
if (write_error < 0)
{
- logprintf (LOG_VERBOSE, _("Failed writing to proxy: %s.\n"),
- strerror (errno));
CLOSE_INVALIDATE (sock);
return WRITEFAILED;
}
if (!head)
{
logprintf (LOG_VERBOSE, _("Failed reading proxy response: %s\n"),
- strerror (errno));
+ fd_errstr (sock));
CLOSE_INVALIDATE (sock);
return HERR;
}
if (write_error < 0)
{
- logprintf (LOG_VERBOSE, _("Failed writing HTTP request: %s.\n"),
- strerror (errno));
CLOSE_INVALIDATE (sock);
request_free (req);
return WRITEFAILED;
else
{
logprintf (LOG_NOTQUIET, _("Read error (%s) in headers.\n"),
- strerror (errno));
+ fd_errstr (sock));
CLOSE_INVALIDATE (sock);
request_free (req);
return HERR;
hs->error = xstrdup (_("(no description)"));
else
hs->error = xstrdup (message);
- xfree (message);
+ xfree_null (message);
type = resp_header_strdup (resp, "Content-Type");
if (type)
hs->len = 0;
hs->res = 0;
xfree_null (type);
- /* Pre-1.10 Wget used CLOSE_INVALIDATE here. Now we trust the
- servers not to send body in response to a HEAD request. If
- you encounter such a server (more likely a broken CGI), use
- `--no-http-keep-alive'. */
- CLOSE_FINISH (sock);
+ if (head_only)
+ /* Pre-1.10 Wget used CLOSE_INVALIDATE here. Now we trust the
+ servers not to send body in response to a HEAD request, and
+ those that do will likely be caught by test_socket_open.
+ If not, they can be worked around using
+ `--no-http-keep-alive'. */
+ CLOSE_FINISH (sock);
+ else if (keep_alive && skip_short_body (sock, contlen))
+ /* Successfully skipped the body; also keep using the socket. */
+ CLOSE_FINISH (sock);
+ else
+ CLOSE_INVALIDATE (sock);
return RETRFINISHED;
}
/* Download the request body. */
flags = 0;
- if (keep_alive)
+ if (contlen != -1)
+ /* If content-length is present, read that much; otherwise, read
+ until EOF. The HTTP spec doesn't require the server to
+ actually close the connection when it's done sending data. */
flags |= rb_read_exactly;
if (hs->restval > 0 && contrange == 0)
/* If the server ignored our range request, instruct fd_read_body
if (hs->res >= 0)
CLOSE_FINISH (sock);
else
- CLOSE_INVALIDATE (sock);
+ {
+ if (hs->res < 0)
+ hs->rderrmsg = xstrdup (fd_errstr (sock));
+ CLOSE_INVALIDATE (sock);
+ }
- {
- /* Close or flush the file. We have to be careful to check for
- error here. Checking the result of fwrite() is not enough --
- errors could go unnoticed! */
- int flush_res;
- if (!output_stream)
- flush_res = fclose (fp);
- else
- flush_res = fflush (fp);
- if (flush_res == EOF)
- hs->res = -2;
- }
+ if (!output_stream)
+ fclose (fp);
if (hs->res == -2)
return FWRITEERR;
return RETRFINISHED;
if ((tmr != (time_t) (-1))
&& !opt.spider
&& ((hstat.len == hstat.contlen) ||
- ((hstat.res == 0) &&
- ((hstat.contlen == -1) ||
- (hstat.len >= hstat.contlen && !opt.kill_longer)))))
+ ((hstat.res == 0) && (hstat.contlen == -1))))
{
/* #### This code repeats in http.c and ftp.c. Move it to a
function! */
free_hstat (&hstat);
continue;
}
- else if (!opt.kill_longer) /* meaning we got more than expected */
- {
- logprintf (LOG_VERBOSE,
- _("%s (%s) - `%s' saved [%s/%s]\n\n"),
- tms, tmrate, locf,
- number_to_static_string (hstat.len),
- number_to_static_string (hstat.contlen));
- logprintf (LOG_NONVERBOSE,
- "%s URL:%s [%s/%s] -> \"%s\" [%d]\n",
- tms, u->url,
- number_to_static_string (hstat.len),
- number_to_static_string (hstat.contlen),
- locf, count);
- ++opt.numurls;
- total_downloaded_bytes += hstat.len;
-
- /* Remember that we downloaded the file for later ".orig" code. */
- if (*dt & ADDED_HTML_EXTENSION)
- downloaded_file(FILE_DOWNLOADED_AND_HTML_EXTENSION_ADDED, locf);
- else
- downloaded_file(FILE_DOWNLOADED_NORMALLY, locf);
-
- free_hstat (&hstat);
- xfree_null (dummy);
- return RETROK;
- }
- else /* the same, but not accepted */
- {
- logprintf (LOG_VERBOSE,
- _("%s (%s) - Connection closed at byte %s/%s. "),
- tms, tmrate,
- number_to_static_string (hstat.len),
- number_to_static_string (hstat.contlen));
- printwhat (count, opt.ntry);
- free_hstat (&hstat);
- continue;
- }
+ else
+ /* Getting here would mean reading more data than
+ requested with content-length, which we never do. */
+ abort ();
}
else /* now hstat.res can only be -1 */
{
logprintf (LOG_VERBOSE,
_("%s (%s) - Read error at byte %s (%s)."),
tms, tmrate, number_to_static_string (hstat.len),
- strerror (errno));
+ hstat.rderrmsg);
printwhat (count, opt.ntry);
free_hstat (&hstat);
continue;
tms, tmrate,
number_to_static_string (hstat.len),
number_to_static_string (hstat.contlen),
- strerror (errno));
+ hstat.rderrmsg);
printwhat (count, opt.ntry);
free_hstat (&hstat);
continue;
buffer of 33 writable characters (32 for hex digits plus one for
zero termination). */
static void
-dump_hash (unsigned char *buf, const unsigned char *hash)
+dump_hash (char *buf, const unsigned char *hash)
{
int i;
{
ALLOCA_MD5_CONTEXT (ctx);
unsigned char hash[MD5_HASHLEN];
- unsigned char a1buf[MD5_HASHLEN * 2 + 1], a2buf[MD5_HASHLEN * 2 + 1];
- unsigned char response_digest[MD5_HASHLEN * 2 + 1];
+ char a1buf[MD5_HASHLEN * 2 + 1], a2buf[MD5_HASHLEN * 2 + 1];
+ char response_digest[MD5_HASHLEN * 2 + 1];
/* A1BUF = H(user ":" realm ":" password) */
gen_md5_init (ctx);
/* RESPONSE_DIGEST = H(A1BUF ":" nonce ":" A2BUF) */
gen_md5_init (ctx);
- gen_md5_update (a1buf, MD5_HASHLEN * 2, ctx);
+ gen_md5_update ((unsigned char *)a1buf, MD5_HASHLEN * 2, ctx);
gen_md5_update ((unsigned char *)":", 1, ctx);
gen_md5_update ((unsigned char *)nonce, strlen (nonce), ctx);
gen_md5_update ((unsigned char *)":", 1, ctx);
- gen_md5_update (a2buf, MD5_HASHLEN * 2, ctx);
+ gen_md5_update ((unsigned char *)a2buf, MD5_HASHLEN * 2, ctx);
gen_md5_finish (ctx, hash);
dump_hash (response_digest, hash);
projects / wget / blobdiff