/* HTTP support.
- Copyright (C) 2005 Free Software Foundation, Inc.
+ Copyright (C) 1996-2005 Free Software Foundation, Inc.
This file is part of GNU Wget.
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
-along with Wget; if not, write to the Free Software
-Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+along with Wget; if not, write to the Free Software Foundation, Inc.,
+51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
In addition, as a special exception, the Free Software Foundation
gives permission to link the code of its release of Wget with the
#include <assert.h>
#include <errno.h>
#include <time.h>
+#include <locale.h>
#include "wget.h"
+#include "http.h"
#include "utils.h"
#include "url.h"
#include "host.h"
extern char *version_string;
-extern FILE *output_stream;
-extern bool output_stream_regular;
-
#ifndef MIN
# define MIN(x, y) ((x) > (y) ? (y) : (x))
#endif
write_error = fd_write (fd, request_string, size - 1, -1);
if (write_error < 0)
logprintf (LOG_VERBOSE, _("Failed writing HTTP request: %s.\n"),
- strerror (errno));
+ fd_errstr (fd));
return write_error;
}
return 0;
}
\f
+/* Determine whether [START, PEEKED + PEEKLEN) contains an empty line.
+ If so, return the pointer to the position after the line, otherwise
+ return NULL. This is used as callback to fd_read_hunk. The data
+ between START and PEEKED has been read and cannot be "unread"; the
+ data after PEEKED has only been peeked. */
+
static const char *
-response_head_terminator (const char *hunk, int oldlen, int peeklen)
+response_head_terminator (const char *start, const char *peeked, int peeklen)
{
- const char *start, *end;
+ const char *p, *end;
/* If at first peek, verify whether HUNK starts with "HTTP". If
not, this is a HTTP/0.9 request and we must bail out without
reading anything. */
- if (oldlen == 0 && 0 != memcmp (hunk, "HTTP", MIN (peeklen, 4)))
- return hunk;
-
- if (oldlen < 4)
- start = hunk;
- else
- start = hunk + oldlen - 4;
- end = hunk + oldlen + peeklen;
-
- for (; start < end - 1; start++)
- if (*start == '\n')
+ if (start == peeked && 0 != memcmp (start, "HTTP", MIN (peeklen, 4)))
+ return start;
+
+ /* Look for "\n[\r]\n", and return the following position if found.
+ Start two chars before the current to cover the possibility that
+ part of the terminator (e.g. "\n\r") arrived in the previous
+ batch. */
+ p = peeked - start < 2 ? start : peeked - 2;
+ end = peeked + peeklen;
+
+ /* Check for \n\r\n or \n\n anywhere in [p, end-2). */
+ for (; p < end - 2; p++)
+ if (*p == '\n')
{
- if (start < end - 2
- && start[1] == '\r'
- && start[2] == '\n')
- return start + 3;
- if (start[1] == '\n')
- return start + 2;
+ if (p[1] == '\r' && p[2] == '\n')
+ return p + 3;
+ else if (p[1] == '\n')
+ return p + 2;
}
+ /* p==end-2: check for \n\n directly preceding END. */
+ if (p[0] == '\n' && p[1] == '\n')
+ return p + 2;
+
return NULL;
}
-/* The maximum size of a single HTTP response we care to read. This
- is not meant to impose an arbitrary limit, but to protect the user
- from Wget slurping up available memory upon encountering malicious
- or buggy server output. Define it to 0 to remove the limit. */
+/* The maximum size of a single HTTP response we care to read. Rather
+ than being a limit of the reader implementation, this limit
+ prevents Wget from slurping all available memory upon encountering
+ malicious or buggy server output, thus protecting the user. Define
+ it to 0 to remove the limit. */
#define HTTP_RESPONSE_MAX_SIZE 65536
/* Don't normally report the error since this is an
optimization that should be invisible to the user. */
DEBUGP (("] aborting (%s).\n",
- ret < 0 ? strerror (errno) : "EOF received"));
+ ret < 0 ? fd_errstr (fd) : "EOF received"));
return false;
}
contlen -= ret;
}
/* Finally, check whether the connection is still open. This is
- important because most server implement a liberal (short) timeout
+ important because most servers implement liberal (short) timeout
on persistent connections. Wget can of course always reconnect
if the connection doesn't work out, but it's nicer to know in
advance. This test is a logical followup of the first test, but
- is "expensive" and therefore placed at the end of the list. */
+ is "expensive" and therefore placed at the end of the list.
+
+ (Current implementation of test_socket_open has a nice side
+ effect that it treats sockets with pending data as "closed".
+ This is exactly what we want: if a broken server sends message
+ body in response to HEAD, or if it sends more than conent-length
+ data, we won't reuse the corrupted connection.) */
if (!test_socket_open (pconn.socket))
{
wgint contlen; /* expected length */
wgint restval; /* the restart value */
int res; /* the result of last read */
+ char *rderrmsg; /* error message from read error */
char *newloc; /* new location (redirection) */
char *remote_time; /* remote time-stamp string */
char *error; /* textual HTTP error */
xfree_null (hs->newloc);
xfree_null (hs->remote_time);
xfree_null (hs->error);
+ xfree_null (hs->rderrmsg);
/* Guard against being called twice. */
hs->newloc = NULL;
static char *basic_authentication_encode (const char *, const char *);
static bool known_authentication_scheme_p (const char *, const char *);
-time_t http_atotm (const char *);
-
#define BEGINS_WITH(line, string_constant) \
(!strncasecmp (line, string_constant, sizeof (string_constant) - 1) \
&& (ISSPACE (line[sizeof (string_constant) - 1]) \
hs->len = 0;
hs->contlen = -1;
hs->res = -1;
+ hs->rderrmsg = NULL;
hs->newloc = NULL;
hs->remote_time = NULL;
hs->error = NULL;
request_set_header (req, "Proxy-Authorization", proxyauth, rel_value);
}
+ /* Generate the Host header, HOST:PORT. Take into account that:
+
+ - Broken server-side software often doesn't recognize the PORT
+ argument, so we must generate "Host: www.server.com" instead of
+ "Host: www.server.com:80" (and likewise for https port).
+
+ - IPv6 addresses contain ":", so "Host: 3ffe:8100:200:2::2:1234"
+ becomes ambiguous and needs to be rewritten as "Host:
+ [3ffe:8100:200:2::2]:1234". */
{
- /* Whether we need to print the host header with braces around
- host, e.g. "Host: [3ffe:8100:200:2::2]:1234" instead of the
- usual "Host: symbolic-name:1234". */
- bool squares = strchr (u->host, ':') != NULL;
- if (u->port == scheme_default_port (u->scheme))
- request_set_header (req, "Host",
- aprintf (squares ? "[%s]" : "%s", u->host),
- rel_value);
- else
- request_set_header (req, "Host",
- aprintf (squares ? "[%s]:%d" : "%s:%d",
- u->host, u->port),
- rel_value);
+ /* Formats arranged for hfmt[add_port][add_squares]. */
+ static const char *hfmt[][2] = {
+ { "%s", "[%s]" }, { "%s:%d", "[%s]:%d" }
+ };
+ int add_port = u->port != scheme_default_port (u->scheme);
+ int add_squares = strchr (u->host, ':') != NULL;
+ request_set_header (req, "Host",
+ aprintf (hfmt[add_port][add_squares], u->host, u->port),
+ rel_value);
}
if (!inhibit_keep_alive)
request_free (connreq);
if (write_error < 0)
{
- logprintf (LOG_VERBOSE, _("Failed writing to proxy: %s.\n"),
- strerror (errno));
CLOSE_INVALIDATE (sock);
return WRITEFAILED;
}
if (!head)
{
logprintf (LOG_VERBOSE, _("Failed reading proxy response: %s\n"),
- strerror (errno));
+ fd_errstr (sock));
CLOSE_INVALIDATE (sock);
return HERR;
}
if (write_error < 0)
{
- logprintf (LOG_VERBOSE, _("Failed writing HTTP request: %s.\n"),
- strerror (errno));
CLOSE_INVALIDATE (sock);
request_free (req);
return WRITEFAILED;
else
{
logprintf (LOG_NOTQUIET, _("Read error (%s) in headers.\n"),
- strerror (errno));
+ fd_errstr (sock));
CLOSE_INVALIDATE (sock);
request_free (req);
return HERR;
hs->error = xstrdup (_("(no description)"));
else
hs->error = xstrdup (message);
- xfree (message);
+ xfree_null (message);
type = resp_header_strdup (resp, "Content-Type");
if (type)
hs->len = 0;
hs->res = 0;
xfree_null (type);
- /* Pre-1.10 Wget used CLOSE_INVALIDATE here. Now we trust the
- servers not to send body in response to a HEAD request. If
- you encounter such a server (more likely a broken CGI), use
- `--no-http-keep-alive'. */
- CLOSE_FINISH (sock);
+ if (head_only)
+ /* Pre-1.10 Wget used CLOSE_INVALIDATE here. Now we trust the
+ servers not to send body in response to a HEAD request, and
+ those that do will likely be caught by test_socket_open.
+ If not, they can be worked around using
+ `--no-http-keep-alive'. */
+ CLOSE_FINISH (sock);
+ else if (keep_alive && skip_short_body (sock, contlen))
+ /* Successfully skipped the body; also keep using the socket. */
+ CLOSE_FINISH (sock);
+ else
+ CLOSE_INVALIDATE (sock);
return RETRFINISHED;
}
else
fp = output_stream;
- /* #### This confuses the timestamping code that checks for file
- size. Maybe we should save some additional information? */
- if (opt.save_headers)
+ /* This confuses the timestamping code that checks for file size.
+ #### The timestamping code should be smarter about file size. */
+ if (opt.save_headers && hs->restval == 0)
fwrite (head, 1, strlen (head), fp);
/* Now we no longer need to store the response header. */
/* Download the request body. */
flags = 0;
- if (keep_alive)
+ if (contlen != -1)
+ /* If content-length is present, read that much; otherwise, read
+ until EOF. The HTTP spec doesn't require the server to
+ actually close the connection when it's done sending data. */
flags |= rb_read_exactly;
if (hs->restval > 0 && contrange == 0)
/* If the server ignored our range request, instruct fd_read_body
if (hs->res >= 0)
CLOSE_FINISH (sock);
else
- CLOSE_INVALIDATE (sock);
+ {
+ if (hs->res < 0)
+ hs->rderrmsg = xstrdup (fd_errstr (sock));
+ CLOSE_INVALIDATE (sock);
+ }
- {
- /* Close or flush the file. We have to be careful to check for
- error here. Checking the result of fwrite() is not enough --
- errors could go unnoticed! */
- int flush_res;
- if (!output_stream)
- flush_res = fclose (fp);
- else
- flush_res = fflush (fp);
- if (flush_res == EOF)
- hs->res = -2;
- }
+ if (!output_stream)
+ fclose (fp);
if (hs->res == -2)
return FWRITEERR;
return RETRFINISHED;
if ((tmr != (time_t) (-1))
&& !opt.spider
&& ((hstat.len == hstat.contlen) ||
- ((hstat.res == 0) &&
- ((hstat.contlen == -1) ||
- (hstat.len >= hstat.contlen && !opt.kill_longer)))))
+ ((hstat.res == 0) && (hstat.contlen == -1))))
{
/* #### This code repeats in http.c and ftp.c. Move it to a
function! */
free_hstat (&hstat);
continue;
}
- else if (!opt.kill_longer) /* meaning we got more than expected */
- {
- logprintf (LOG_VERBOSE,
- _("%s (%s) - `%s' saved [%s/%s]\n\n"),
- tms, tmrate, locf,
- number_to_static_string (hstat.len),
- number_to_static_string (hstat.contlen));
- logprintf (LOG_NONVERBOSE,
- "%s URL:%s [%s/%s] -> \"%s\" [%d]\n",
- tms, u->url,
- number_to_static_string (hstat.len),
- number_to_static_string (hstat.contlen),
- locf, count);
- ++opt.numurls;
- total_downloaded_bytes += hstat.len;
-
- /* Remember that we downloaded the file for later ".orig" code. */
- if (*dt & ADDED_HTML_EXTENSION)
- downloaded_file(FILE_DOWNLOADED_AND_HTML_EXTENSION_ADDED, locf);
- else
- downloaded_file(FILE_DOWNLOADED_NORMALLY, locf);
-
- free_hstat (&hstat);
- xfree_null (dummy);
- return RETROK;
- }
- else /* the same, but not accepted */
- {
- logprintf (LOG_VERBOSE,
- _("%s (%s) - Connection closed at byte %s/%s. "),
- tms, tmrate,
- number_to_static_string (hstat.len),
- number_to_static_string (hstat.contlen));
- printwhat (count, opt.ntry);
- free_hstat (&hstat);
- continue;
- }
+ else
+ /* Getting here would mean reading more data than
+ requested with content-length, which we never do. */
+ abort ();
}
else /* now hstat.res can only be -1 */
{
logprintf (LOG_VERBOSE,
_("%s (%s) - Read error at byte %s (%s)."),
tms, tmrate, number_to_static_string (hstat.len),
- strerror (errno));
+ hstat.rderrmsg);
printwhat (count, opt.ntry);
free_hstat (&hstat);
continue;
tms, tmrate,
number_to_static_string (hstat.len),
number_to_static_string (hstat.contlen),
- strerror (errno));
+ hstat.rderrmsg);
printwhat (count, opt.ntry);
free_hstat (&hstat);
continue;
return TRYLIMEXC;
}
\f
-/* Converts struct tm to time_t, assuming the data in tm is UTC rather
- than local timezone.
-
- mktime is similar but assumes struct tm, also known as the
- "broken-down" form of time, is in local time zone. mktime_from_utc
- uses mktime to make the conversion understanding that an offset
- will be introduced by the local time assumption.
-
- mktime_from_utc then measures the introduced offset by applying
- gmtime to the initial result and applying mktime to the resulting
- "broken-down" form. The difference between the two mktime results
- is the measured offset which is then subtracted from the initial
- mktime result to yield a calendar time which is the value returned.
-
- tm_isdst in struct tm is set to 0 to force mktime to introduce a
- consistent offset (the non DST offset) since tm and tm+o might be
- on opposite sides of a DST change.
-
- Some implementations of mktime return -1 for the nonexistent
- localtime hour at the beginning of DST. In this event, use
- mktime(tm - 1hr) + 3600.
-
- Schematically
- mktime(tm) --> t+o
- gmtime(t+o) --> tm+o
- mktime(tm+o) --> t+2o
- t+o - (t+2o - t+o) = t
-
- Note that glibc contains a function of the same purpose named
- `timegm' (reverse of gmtime). But obviously, it is not universally
- available, and unfortunately it is not straightforwardly
- extractable for use here. Perhaps configure should detect timegm
- and use it where available.
-
- Contributed by Roger Beeman <beeman@cisco.com>, with the help of
- Mark Baushke <mdb@cisco.com> and the rest of the Gurus at CISCO.
- Further improved by Roger with assistance from Edward J. Sabol
- based on input by Jamie Zawinski. */
-
-static time_t
-mktime_from_utc (struct tm *t)
-{
- time_t tl, tb;
- struct tm *tg;
-
- tl = mktime (t);
- if (tl == -1)
- {
- t->tm_hour--;
- tl = mktime (t);
- if (tl == -1)
- return -1; /* can't deal with output from strptime */
- tl += 3600;
- }
- tg = gmtime (&tl);
- tg->tm_isdst = 0;
- tb = mktime (tg);
- if (tb == -1)
- {
- tg->tm_hour--;
- tb = mktime (tg);
- if (tb == -1)
- return -1; /* can't deal with output from gmtime */
- tb += 3600;
- }
- return (tl - (tb - tl));
-}
-
/* Check whether the result of strptime() indicates success.
strptime() returns the pointer to how far it got to in the string.
The processing has been successful if the string is at `GMT' or
(used in Set-Cookie, defined in the
Netscape cookie specification.) */
};
+ const char *oldlocale;
int i;
+ time_t ret = (time_t) -1;
+
+ /* Solaris strptime fails to recognize English month names in
+ non-English locales, which we work around by temporarily setting
+ locale to C before invoking strptime. */
+ oldlocale = setlocale (LC_TIME, NULL);
+ setlocale (LC_TIME, "C");
for (i = 0; i < countof (time_formats); i++)
{
/* Some versions of strptime use the existing contents of struct
tm to recalculate the date according to format. Zero it out
- to prevent garbage from the stack influencing strptime. */
+ to prevent stack garbage from influencing strptime. */
xzero (t);
- /* Solaris strptime fails to recognize English month names in
- non-English locales, which we work around by not setting the
- LC_TIME category. Another way would be to temporarily set
- locale to C before invoking strptime, but that's slow and
- messy. GNU strptime does not have this problem because it
- recognizes English month names along with the local ones. */
-
if (check_end (strptime (time_string, time_formats[i], &t)))
- return mktime_from_utc (&t);
+ {
+ ret = timegm (&t);
+ break;
+ }
}
- /* All formats have failed. */
- return -1;
+ /* Restore the previous locale. */
+ setlocale (LC_TIME, oldlocale);
+
+ return ret;
}
\f
/* Authorization support: We support three authorization schemes:
buffer of 33 writable characters (32 for hex digits plus one for
zero termination). */
static void
-dump_hash (unsigned char *buf, const unsigned char *hash)
+dump_hash (char *buf, const unsigned char *hash)
{
int i;
{
ALLOCA_MD5_CONTEXT (ctx);
unsigned char hash[MD5_HASHLEN];
- unsigned char a1buf[MD5_HASHLEN * 2 + 1], a2buf[MD5_HASHLEN * 2 + 1];
- unsigned char response_digest[MD5_HASHLEN * 2 + 1];
+ char a1buf[MD5_HASHLEN * 2 + 1], a2buf[MD5_HASHLEN * 2 + 1];
+ char response_digest[MD5_HASHLEN * 2 + 1];
/* A1BUF = H(user ":" realm ":" password) */
gen_md5_init (ctx);
/* RESPONSE_DIGEST = H(A1BUF ":" nonce ":" A2BUF) */
gen_md5_init (ctx);
- gen_md5_update (a1buf, MD5_HASHLEN * 2, ctx);
+ gen_md5_update ((unsigned char *)a1buf, MD5_HASHLEN * 2, ctx);
gen_md5_update ((unsigned char *)":", 1, ctx);
gen_md5_update ((unsigned char *)nonce, strlen (nonce), ctx);
gen_md5_update ((unsigned char *)":", 1, ctx);
- gen_md5_update (a2buf, MD5_HASHLEN * 2, ctx);
+ gen_md5_update ((unsigned char *)a2buf, MD5_HASHLEN * 2, ctx);
gen_md5_finish (ctx, hash);
dump_hash (response_digest, hash);
projects / wget / blobdiff