/* HTTP support.
- Copyright (C) 1995, 1996, 1997, 1998, 2000 Free Software Foundation, Inc.
+ Copyright (C) 1995, 1996, 1997, 1998, 2000, 2001
+ Free Software Foundation, Inc.
-This file is part of Wget.
+This file is part of GNU Wget.
-This program is free software; you can redistribute it and/or modify
+GNU Wget is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
-This program is distributed in the hope that it will be useful,
+GNU Wget is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
-along with this program; if not, write to the Free Software
+along with Wget; if not, write to the Free Software
Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */
#include <config.h>
keep_alive = 0;
http_keep_alive_1 = http_keep_alive_2 = 0;
- if (opt.cookies)
- cookies = build_cookies_request (u->host, u->port, u->path,
- u->proto == URLHTTPS);
-
/* Initialize certain elements of struct http_stat. */
hs->len = 0L;
hs->contlen = -1;
else
request_keep_alive = NULL;
+ if (opt.cookies)
+ cookies = build_cookies_request (ou->host, ou->port, ou->path,
+ ou->proto == URLHTTPS);
+
/* Allocate the memory for the request. */
request = (char *)alloca (strlen (command) + strlen (path)
+ strlen (useragent)
logputs (LOG_NOTQUIET, _("End of file while parsing headers.\n"));
xfree (hdr);
FREE_MAYBE (type);
- FREE_MAYBE (hs->newloc);
FREE_MAYBE (all_headers);
CLOSE_INVALIDATE (sock);
return HEOF;
strerror (errno));
xfree (hdr);
FREE_MAYBE (type);
- FREE_MAYBE (hs->newloc);
FREE_MAYBE (all_headers);
CLOSE_INVALIDATE (sock);
return HERR;
if (H_20X (statcode))
*dt |= RETROKF;
+ /* Return if redirected. */
+ if (H_REDIRECTED (statcode) || statcode == HTTP_STATUS_MULTIPLE_CHOICES)
+ {
+ /* RFC2068 says that in case of the 300 (multiple choices)
+ response, the server can output a preferred URL through
+ `Location' header; otherwise, the request should be treated
+ like GET. So, if the location is set, it will be a
+ redirection; otherwise, just proceed normally. */
+ if (statcode == HTTP_STATUS_MULTIPLE_CHOICES && !hs->newloc)
+ *dt |= RETROKF;
+ else
+ {
+ logprintf (LOG_VERBOSE,
+ _("Location: %s%s\n"),
+ hs->newloc ? hs->newloc : _("unspecified"),
+ hs->newloc ? _(" [following]") : "");
+ CLOSE_INVALIDATE (sock); /* would be CLOSE_FINISH, but there
+ might be more bytes in the body. */
+ FREE_MAYBE (type);
+ FREE_MAYBE (all_headers);
+ return NEWLOCATION;
+ }
+ }
+
if (type && !strncasecmp (type, TEXTHTML_S, strlen (TEXTHTML_S)))
*dt |= TEXTHTML;
else
if (opt.always_rest)
{
/* Check for condition #2. */
- if (hs->restval == contlen)
+ if (hs->restval > 0 /* restart was requested. */
+ && contlen != -1 /* we got content-length. */
+ && hs->restval >= contlen /* file fully downloaded
+ or has shrunk. */
+ )
{
logputs (LOG_VERBOSE, _("\
\n The file is already fully retrieved; nothing to do.\n\n"));
hs->len = contlen;
hs->res = 0;
FREE_MAYBE (type);
- FREE_MAYBE (hs->newloc);
FREE_MAYBE (all_headers);
CLOSE_INVALIDATE (sock); /* would be CLOSE_FINISH, but there
might be more bytes in the body. */
- return RETRFINISHED;
+ return RETRUNNEEDED;
}
/* Check for condition #1. */
logprintf (LOG_NOTQUIET,
_("\
\n\
- The server does not support continued download;\n\
- refusing to truncate `%s'.\n\n"), u->local);
+Continued download failed on this file, which conflicts with `-c'.\n\
+Refusing to truncate existing file `%s'.\n\n"), u->local);
+ FREE_MAYBE (type);
+ FREE_MAYBE (all_headers);
+ CLOSE_INVALIDATE (sock);
return CONTNOTSUPPORTED;
}
hs->restval = 0;
}
-
else if (contrange != hs->restval ||
(H_PARTIAL (statcode) && contrange == -1))
{
/* This means the whole request was somehow misunderstood by the
server. Bail out. */
FREE_MAYBE (type);
- FREE_MAYBE (hs->newloc);
FREE_MAYBE (all_headers);
CLOSE_INVALIDATE (sock);
return RANGEERR;
}
hs->contlen = contlen;
- /* Return if redirected. */
- if (H_REDIRECTED (statcode) || statcode == HTTP_STATUS_MULTIPLE_CHOICES)
- {
- /* RFC2068 says that in case of the 300 (multiple choices)
- response, the server can output a preferred URL through
- `Location' header; otherwise, the request should be treated
- like GET. So, if the location is set, it will be a
- redirection; otherwise, just proceed normally. */
- if (statcode == HTTP_STATUS_MULTIPLE_CHOICES && !hs->newloc)
- *dt |= RETROKF;
- else
- {
- logprintf (LOG_VERBOSE,
- _("Location: %s%s\n"),
- hs->newloc ? hs->newloc : _("unspecified"),
- hs->newloc ? _(" [following]") : "");
- CLOSE_INVALIDATE (sock); /* would be CLOSE_FINISH, but there
- might be more bytes in the body. */
- FREE_MAYBE (type);
- FREE_MAYBE (all_headers);
- return NEWLOCATION;
- }
- }
if (opt.verbose)
{
if ((*dt & RETROKF) && !opt.server_response)
here so that we don't go through the hoops if we're just using
FTP or whatever. */
if (opt.cookies && opt.cookies_input && !cookies_loaded_p)
- load_cookies (opt.cookies_input);
+ {
+ load_cookies (opt.cookies_input);
+ cookies_loaded_p = 1;
+ }
*newloc = NULL;
else
locf = opt.output_document;
- /* Yuck. Multiple returns suck. We need to remember to free() the space we
- xmalloc() here before EACH return. This is one reason it's better to set
- flags that influence flow control and then return once at the end. */
- filename_len = strlen(u->local);
- filename_plus_orig_suffix = xmalloc(filename_len + sizeof(".orig"));
+ filename_len = strlen (u->local);
+ filename_plus_orig_suffix = alloca (filename_len + sizeof (".orig"));
if (opt.noclobber && file_exists_p (u->local))
{
&& (!strcmp (suf, "html") || !strcmp (suf, "htm")))
*dt |= TEXTHTML;
xfree (suf);
- xfree (filename_plus_orig_suffix); /* must precede every return! */
/* Another harmless lie: */
return RETROK;
}
in url.c. Replacing sprintf with inline calls to
strcpy() and long_to_string() made a difference.
--hniksic */
- strcpy(filename_plus_orig_suffix, u->local);
- strcpy(filename_plus_orig_suffix + filename_len, ".orig");
+ memcpy (filename_plus_orig_suffix, u->local, filename_len);
+ memcpy (filename_plus_orig_suffix + filename_len,
+ ".orig", sizeof (".orig"));
/* Try to stat() the .orig file. */
- if (stat(filename_plus_orig_suffix, &st) == 0)
+ if (stat (filename_plus_orig_suffix, &st) == 0)
{
local_dot_orig_file_exists = TRUE;
local_filename = filename_plus_orig_suffix;
hstat.restval = 0L;
/* Decide whether or not to restart. */
if (((count > 1 && (*dt & ACCEPTRANGES)) || opt.always_rest)
+ /* #### this calls access() and then stat(); could be optimized. */
&& file_exists_p (locf))
if (stat (locf, &st) == 0 && S_ISREG (st.st_mode))
hstat.restval = st.st_size;
- /* Decide whether to send the no-cache directive. */
- if (u->proxy && (count > 1 || (opt.proxy_cache == 0)))
+
+ /* In `-c' is used and the file is existing and non-empty,
+ refuse to truncate it if the server doesn't support continued
+ downloads. */
+ hstat.no_truncate = 0;
+ if (opt.always_rest && hstat.restval)
+ hstat.no_truncate = 1;
+
+ /* Decide whether to send the no-cache directive. We send it in
+ two cases:
+ a) we're using a proxy, and we're past our first retrieval.
+ Some proxies are notorious for caching incomplete data, so
+ we require a fresh get.
+ b) caching is explicitly inhibited. */
+ if ((u->proxy && count > 1) /* a */
+ || !opt.allow_cache /* b */
+ )
*dt |= SEND_NOCACHE;
else
*dt &= ~SEND_NOCACHE;
else
locf = opt.output_document;
- /* In `-c' is used, check whether the file we're writing to
- exists before we've done anything. If so, we'll refuse to
- truncate it if the server doesn't support continued
- downloads. */
- if (opt.always_rest)
- hstat.no_truncate = file_exists_p (locf);
-
/* Time? */
tms = time_str (NULL);
/* Get the new location (with or without the redirection). */
case SSLERRCTXCREATE: case CONTNOTSUPPORTED:
/* Fatal errors just return from the function. */
FREEHSTAT (hstat);
- xfree (filename_plus_orig_suffix); /* must precede every return! */
return err;
break;
case FWRITEERR: case FOPENERR:
logputs (LOG_VERBOSE, "\n");
logprintf (LOG_NOTQUIET, _("Unable to establish SSL connection.\n"));
FREEHSTAT (hstat);
- xfree (filename_plus_orig_suffix); /* must precede every return! */
return err;
break;
case NEWLOCATION:
logprintf (LOG_NOTQUIET,
_("ERROR: Redirection (%d) without location.\n"),
hstat.statcode);
- xfree (filename_plus_orig_suffix); /* must precede every return! */
return WRONGCODE;
}
FREEHSTAT (hstat);
- xfree (filename_plus_orig_suffix); /* must precede every return! */
return NEWLOCATION;
break;
+ case RETRUNNEEDED:
+ /* The file was already fully retrieved. */
+ FREEHSTAT (hstat);
+ return RETROK;
+ break;
case RETRFINISHED:
/* Deal with you later. */
break;
tms, hstat.statcode, hstat.error);
logputs (LOG_VERBOSE, "\n");
FREEHSTAT (hstat);
- xfree (filename_plus_orig_suffix); /* must precede every return! */
return WRONGCODE;
}
Server file no newer than local file `%s' -- not retrieving.\n\n"),
local_filename);
FREEHSTAT (hstat);
- xfree (filename_plus_orig_suffix); /*must precede every return!*/
return RETROK;
}
else if (tml >= tmr)
if (opt.spider)
{
logprintf (LOG_NOTQUIET, "%d %s\n\n", hstat.statcode, hstat.error);
- xfree (filename_plus_orig_suffix); /* must precede every return! */
return RETROK;
}
else
downloaded_file(FILE_DOWNLOADED_NORMALLY, locf);
- xfree(filename_plus_orig_suffix); /* must precede every return! */
return RETROK;
}
else if (hstat.res == 0) /* No read error */
else
downloaded_file(FILE_DOWNLOADED_NORMALLY, locf);
- xfree (filename_plus_orig_suffix); /* must precede every return! */
return RETROK;
}
else if (hstat.len < hstat.contlen) /* meaning we lost the
else
downloaded_file(FILE_DOWNLOADED_NORMALLY, locf);
- xfree (filename_plus_orig_suffix); /* must precede every return! */
return RETROK;
}
else /* the same, but not accepted */
break;
}
while (!opt.ntry || (count < opt.ntry));
- xfree (filename_plus_orig_suffix); /* must precede every return! */
return TRYLIMEXC;
}
\f
/* Converts struct tm to time_t, assuming the data in tm is UTC rather
- than local timezone (mktime assumes the latter).
+ than local timezone.
+
+ mktime is similar but assumes struct tm, also known as the
+ "broken-down" form of time, is in local time zone. mktime_from_utc
+ uses mktime to make the conversion understanding that an offset
+ will be introduced by the local time assumption.
+
+ mktime_from_utc then measures the introduced offset by applying
+ gmtime to the initial result and applying mktime to the resulting
+ "broken-down" form. The difference between the two mktime results
+ is the measured offset which is then subtracted from the initial
+ mktime result to yield a calendar time which is the value returned.
+
+ tm_isdst in struct tm is set to 0 to force mktime to introduce a
+ consistent offset (the non DST offset) since tm and tm+o might be
+ on opposite sides of a DST change.
+
+ Some implementations of mktime return -1 for the nonexistent
+ localtime hour at the beginning of DST. In this event, use
+ mktime(tm - 1hr) + 3600.
+
+ Schematically
+ mktime(tm) --> t+o
+ gmtime(t+o) --> tm+o
+ mktime(tm+o) --> t+2o
+ t+o - (t+2o - t+o) = t
+
+ Note that glibc contains a function of the same purpose named
+ `timegm' (reverse of gmtime). But obviously, it is not universally
+ available, and unfortunately it is not straightforwardly
+ extractable for use here. Perhaps configure should detect timegm
+ and use it where available.
Contributed by Roger Beeman <beeman@cisco.com>, with the help of
- Mark Baushke <mdb@cisco.com> and the rest of the Gurus at CISCO. */
+ Mark Baushke <mdb@cisco.com> and the rest of the Gurus at CISCO.
+ Further improved by Roger with assistance from Edward J. Sabol
+ based on input by Jamie Zawinski. */
+
static time_t
mktime_from_utc (struct tm *t)
{
time_t tl, tb;
+ struct tm *tg;
tl = mktime (t);
if (tl == -1)
- return -1;
- tb = mktime (gmtime (&tl));
- return (tl <= tb ? (tl + (tl - tb)) : (tl - (tb - tl)));
+ {
+ t->tm_hour--;
+ tl = mktime (t);
+ if (tl == -1)
+ return -1; /* can't deal with output from strptime */
+ tl += 3600;
+ }
+ tg = gmtime (&tl);
+ tg->tm_isdst = 0;
+ tb = mktime (tg);
+ if (tb == -1)
+ {
+ tg->tm_hour--;
+ tb = mktime (tg);
+ if (tb == -1)
+ return -1; /* can't deal with output from gmtime */
+ tb += 3600;
+ }
+ return (tl - (tb - tl));
}
/* Check whether the result of strptime() indicates success.
`+X', or at the end of the string.
In extended regexp parlance, the function returns 1 if P matches
- "^ *(GMT|[+-][0-9]|$)", 0 otherwise. P being NULL (a valid result of
- strptime()) is considered a failure and 0 is returned. */
+ "^ *(GMT|[+-][0-9]|$)", 0 otherwise. P being NULL (which strptime
+ can return) is considered a failure and 0 is returned. */
static int
check_end (const char *p)
{
return 0;
}
-/* Convert TIME_STRING time to time_t. TIME_STRING can be in any of
- the three formats RFC2068 allows the HTTP servers to emit --
- RFC1123-date, RFC850-date or asctime-date. Timezones are ignored,
- and should be GMT.
-
- We use strptime() to recognize various dates, which makes it a
- little bit slacker than the RFC1123/RFC850/asctime (e.g. it always
- allows shortened dates and months, one-digit days, etc.). It also
- allows more than one space anywhere where the specs require one SP.
- The routine should probably be even more forgiving (as recommended
- by RFC2068), but I do not have the time to write one.
-
- Return the computed time_t representation, or -1 if all the
- schemes fail.
-
- Needless to say, what we *really* need here is something like
- Marcus Hennecke's atotm(), which is forgiving, fast, to-the-point,
- and does not use strptime(). atotm() is to be found in the sources
- of `phttpd', a little-known HTTP server written by Peter Erikson. */
+/* Convert the textual specification of time in TIME_STRING to the
+ number of seconds since the Epoch.
+
+ TIME_STRING can be in any of the three formats RFC2068 allows the
+ HTTP servers to emit -- RFC1123-date, RFC850-date or asctime-date.
+ Timezones are ignored, and should be GMT.
+
+ Return the computed time_t representation, or -1 if the conversion
+ fails.
+
+ This function uses strptime with various string formats for parsing
+ TIME_STRING. This results in a parser that is not as lenient in
+ interpreting TIME_STRING as I would like it to be. Being based on
+ strptime, it always allows shortened months, one-digit days, etc.,
+ but due to the multitude of formats in which time can be
+ represented, an ideal HTTP time parser would be even more
+ forgiving. It should completely ignore things like week days and
+ concentrate only on the various forms of representing years,
+ months, days, hours, minutes, and seconds. For example, it would
+ be nice if it accepted ISO 8601 out of the box.
+
+ I've investigated free and PD code for this purpose, but none was
+ usable. getdate was big and unwieldy, and had potential copyright
+ issues, or so I was informed. Dr. Marcus Hennecke's atotm(),
+ distributed with phttpd, is excellent, but we cannot use it because
+ it is not assigned to the FSF. So I stuck it with strptime. */
+
time_t
http_atotm (char *time_string)
{
+ /* NOTE: Solaris strptime man page claims that %n and %t match white
+ space, but that's not universally available. Instead, we simply
+ use ` ' to mean "skip all WS", which works under all strptime
+ implementations I've tested. */
+
+ static const char *time_formats[] = {
+ "%a, %d %b %Y %T", /* RFC1123: Thu, 29 Jan 1998 22:12:57 */
+ "%A, %d-%b-%y %T", /* RFC850: Thursday, 29-Jan-98 22:12:57 */
+ "%a, %d-%b-%Y %T", /* pseudo-RFC850: Thu, 29-Jan-1998 22:12:57
+ (google.com uses this for their cookies.) */
+ "%a %b %d %T %Y" /* asctime: Thu Jan 29 22:12:57 1998 */
+ };
+
+ int i;
struct tm t;
- /* Roger Beeman says: "This function dynamically allocates struct tm
- t, but does no initialization. The only field that actually
- needs initialization is tm_isdst, since the others will be set by
- strptime. Since strptime does not set tm_isdst, it will return
- the data structure with whatever data was in tm_isdst to begin
- with. For those of us in timezones where DST can occur, there
- can be a one hour shift depending on the previous contents of the
- data area where the data structure is allocated." */
- t.tm_isdst = -1;
+ /* According to Roger Beeman, we need to initialize tm_isdst, since
+ strptime won't do it. */
+ t.tm_isdst = 0;
/* Note that under foreign locales Solaris strptime() fails to
- recognize English dates, which renders this function useless. I
- assume that other non-GNU strptime's are plagued by the same
- disease. We solve this by setting only LC_MESSAGES in
- i18n_initialize(), instead of LC_ALL.
+ recognize English dates, which renders this function useless. We
+ solve this by being careful not to affect LC_TIME when
+ initializing locale.
- Another solution could be to temporarily set locale to C, invoke
+ Another solution would be to temporarily set locale to C, invoke
strptime(), and restore it back. This is slow and dirty,
however, and locale support other than LC_MESSAGES can mess other
things, so I rather chose to stick with just setting LC_MESSAGES.
- Also note that none of this is necessary under GNU strptime(),
- because it recognizes both international and local dates. */
-
- /* NOTE: We don't use `%n' for white space, as OSF's strptime uses
- it to eat all white space up to (and including) a newline, and
- the function fails if there is no newline (!).
-
- Let's hope all strptime() implementations use ` ' to skip *all*
- whitespace instead of just one (it works that way on all the
- systems I've tested it on). */
-
- /* RFC1123: Thu, 29 Jan 1998 22:12:57 */
- if (check_end (strptime (time_string, "%a, %d %b %Y %T", &t)))
- return mktime_from_utc (&t);
- /* RFC850: Thursday, 29-Jan-98 22:12:57 */
- if (check_end (strptime (time_string, "%A, %d-%b-%y %T", &t)))
- return mktime_from_utc (&t);
- /* pseudo-RFC850: Thu, 29-Jan-1998 22:12:57
- (google.com uses this for their cookies.)*/
- if (check_end (strptime (time_string, "%a, %d-%b-%Y %T", &t)))
- return mktime_from_utc (&t);
- /* asctime: Thu Jan 29 22:12:57 1998 */
- if (check_end (strptime (time_string, "%a %b %d %T %Y", &t)))
- return mktime_from_utc (&t);
- /* Failure. */
+ GNU strptime does not have this problem because it recognizes
+ both international and local dates. */
+
+ for (i = 0; i < ARRAY_SIZE (time_formats); i++)
+ if (check_end (strptime (time_string, time_formats[i], &t)))
+ return mktime_from_utc (&t);
+
+ /* All formats have failed. */
return -1;
}
\f