/* File retrieval.
- Copyright (C) 1996-2005 Free Software Foundation, Inc.
+ Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
+ 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
+ Inc.
This file is part of GNU Wget.
GNU Wget is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or (at
+the Free Software Foundation; either version 3 of the License, or (at
your option) any later version.
GNU Wget is distributed in the hope that it will be useful,
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
-along with Wget; if not, write to the Free Software Foundation, Inc.,
-51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+along with Wget. If not, see <http://www.gnu.org/licenses/>.
-In addition, as a special exception, the Free Software Foundation
-gives permission to link the code of its release of Wget with the
-OpenSSL project's "OpenSSL" library (or with modified versions of it
-that use the same license as the "OpenSSL" library), and distribute
-the linked executables. You must obey the GNU General Public License
-in all respects for all of the code used other than "OpenSSL". If you
-modify this file, you may extend this exception to your version of the
-file, but you are not obligated to do so. If you do not wish to do
-so, delete this exception statement from your version. */
+Additional permission under GNU GPL version 3 section 7
-#include <config.h>
+If you modify this program, or any covered work, by linking or
+combining it with the OpenSSL project's OpenSSL library (or a
+modified version of that library), containing parts covered by the
+terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
+grants you additional permission to convey the resulting work.
+Corresponding Source for a non-source form of such a combination
+shall include the source code for the parts of OpenSSL used as well
+as that of the covered work. */
+
+#include "wget.h"
#include <stdio.h>
#include <stdlib.h>
-#ifdef HAVE_UNISTD_H
-# include <unistd.h>
-#endif /* HAVE_UNISTD_H */
+#include <unistd.h>
#include <errno.h>
#include <string.h>
#include <assert.h>
+#ifdef VMS
+# include <unixio.h> /* For delete(). */
+#endif
-#include "wget.h"
+#include "exits.h"
#include "utils.h"
#include "retr.h"
#include "progress.h"
#include "hash.h"
#include "convert.h"
#include "ptimer.h"
+#include "html-url.h"
+#include "iri.h"
/* Total size of downloaded files. Used to enforce quota. */
SUM_SIZE_INT total_downloaded_bytes;
-/* Total download time in milliseconds. */
+/* Total download time in seconds. */
double total_download_time;
/* If non-NULL, the stream to which output should be written. This
static void
limit_bandwidth_reset (void)
{
- limit_data.chunk_bytes = 0;
- limit_data.chunk_start = 0;
- limit_data.sleep_adjust = 0;
+ xzero (limit_data);
}
/* Limit the bandwidth by pausing the download for an amount of time.
/* Calculate the amount of time we expect downloading the chunk
should take. If in reality it took less time, sleep to
compensate for the difference. */
- expected = 1000.0 * limit_data.chunk_bytes / opt.limit_rate;
+ expected = (double) limit_data.chunk_bytes / opt.limit_rate;
if (expected > delta_t)
{
double slp = expected - delta_t + limit_data.sleep_adjust;
double t0, t1;
- if (slp < 200)
- {
- DEBUGP (("deferring a %.2f ms sleep (%s/%.2f).\n",
- slp, number_to_static_string (limit_data.chunk_bytes),
- delta_t));
- return;
- }
+ if (slp < 0.2)
+ {
+ DEBUGP (("deferring a %.2f ms sleep (%s/%.2f).\n",
+ slp * 1000, number_to_static_string (limit_data.chunk_bytes),
+ delta_t));
+ return;
+ }
DEBUGP (("\nsleeping %.2f ms for %s bytes, adjust %.2f ms\n",
- slp, number_to_static_string (limit_data.chunk_bytes),
- limit_data.sleep_adjust));
+ slp * 1000, number_to_static_string (limit_data.chunk_bytes),
+ limit_data.sleep_adjust));
t0 = ptimer_read (timer);
- xsleep (slp / 1000);
+ xsleep (slp);
t1 = ptimer_measure (timer);
/* Due to scheduling, we probably slept slightly longer (or
- shorter) than desired. Calculate the difference between the
- desired and the actual sleep, and adjust the next sleep by
- that amount. */
+ shorter) than desired. Calculate the difference between the
+ desired and the actual sleep, and adjust the next sleep by
+ that amount. */
limit_data.sleep_adjust = slp - (t1 - t0);
/* If sleep_adjust is very large, it's likely due to suspension
- and not clock inaccuracy. Don't enforce those. */
- if (limit_data.sleep_adjust > 500)
- limit_data.sleep_adjust = 500;
- else if (limit_data.sleep_adjust < -500)
- limit_data.sleep_adjust = -500;
+ and not clock inaccuracy. Don't enforce those. */
+ if (limit_data.sleep_adjust > 0.5)
+ limit_data.sleep_adjust = 0.5;
+ else if (limit_data.sleep_adjust < -0.5)
+ limit_data.sleep_adjust = -0.5;
}
limit_data.chunk_bytes = 0;
/* Write data in BUF to OUT. However, if *SKIP is non-zero, skip that
amount of data and decrease SKIP. Increment *TOTAL by the amount
- of data written. */
+ of data written. If OUT2 is not NULL, also write BUF to OUT2.
+ In case of error writing to OUT, -1 is returned. In case of error
+ writing to OUT2, -2 is returned. Return 1 if the whole BUF was
+ skipped. */
static int
-write_data (FILE *out, const char *buf, int bufsize, wgint *skip,
- wgint *written)
+write_data (FILE *out, FILE *out2, const char *buf, int bufsize,
+ wgint *skip, wgint *written)
{
- if (!out)
+ if (out == NULL && out2 == NULL)
return 1;
if (*skip > bufsize)
{
bufsize -= *skip;
*skip = 0;
if (bufsize == 0)
- return 1;
+ return 1;
}
- fwrite (buf, 1, bufsize, out);
+ if (out != NULL)
+ fwrite (buf, 1, bufsize, out);
+ if (out2 != NULL)
+ fwrite (buf, 1, bufsize, out2);
*written += bufsize;
/* Immediately flush the downloaded data. This should not hinder
performance: fast downloads will arrive in large 16K chunks
(which stdio would write out immediately anyway), and slow
downloads wouldn't be limited by disk speed. */
- fflush (out);
- return !ferror (out);
+
+ /* 2005-04-20 SMS.
+ Perhaps it shouldn't hinder performance, but it sure does, at least
+ on VMS (more than 2X). Rather than speculate on what it should or
+ shouldn't do, it might make more sense to test it. Even better, it
+ might be nice to explain what possible benefit it could offer, as
+ it appears to be a clear invitation to poor performance with no
+ actual justification. (Also, why 16K? Anyone test other values?)
+ */
+#ifndef __VMS
+ if (out != NULL)
+ fflush (out);
+ if (out2 != NULL)
+ fflush (out2);
+#endif /* ndef __VMS */
+ if (out != NULL && ferror (out))
+ return -1;
+ else if (out2 != NULL && ferror (out2))
+ return -2;
+ else
+ return 0;
}
/* Read the contents of file descriptor FD until it the connection
is incremented by the amount of data read from the network. If
QTYWRITTEN is non-NULL, the value it points to is incremented by
the amount of data written to disk. The time it took to download
- the data (in milliseconds) is stored to ELAPSED.
+ the data is stored to ELAPSED.
+
+ If OUT2 is non-NULL, the contents is also written to OUT2.
+ OUT2 will get an exact copy of the response: if this is a chunked
+ response, everything -- including the chunk headers -- is written
+ to OUT2. (OUT will only get the unchunked response.)
The function exits and returns the amount of data read. In case of
error while reading data, -1 is returned. In case of error while
- writing data, -2 is returned. */
+ writing data to OUT, -2 is returned. In case of error while writing
+ data to OUT2, -3 is returned. */
int
fd_read_body (int fd, FILE *out, wgint toread, wgint startpos,
- wgint *qtyread, wgint *qtywritten, double *elapsed, int flags)
+ wgint *qtyread, wgint *qtywritten, double *elapsed, int flags,
+ FILE *out2)
{
int ret = 0;
-
- static char dlbuf[16384];
- int dlbufsize = sizeof (dlbuf);
+#undef max
+#define max(a,b) ((a) > (b) ? (a) : (b))
+ int dlbufsize = max (BUFSIZ, 8 * 1024);
+ char *dlbuf = xmalloc (dlbufsize);
struct ptimer *timer = NULL;
double last_successful_read_tm = 0;
bool progress_interactive = false;
bool exact = !!(flags & rb_read_exactly);
+
+ /* Used only by HTTP/HTTPS chunked transfer encoding. */
+ bool chunked = flags & rb_chunked_transfer_encoding;
wgint skip = 0;
/* How much data we've read/written. */
wgint sum_read = 0;
wgint sum_written = 0;
+ wgint remaining_chunk_size = 0;
if (flags & rb_skip_startpos)
skip = startpos;
if (opt.verbose)
{
/* If we're skipping STARTPOS bytes, pass 0 as the INITIAL
- argument to progress_create because the indicator doesn't
- (yet) know about "skipping" data. */
- progress = progress_create (skip ? 0 : startpos, startpos + toread);
+ argument to progress_create because the indicator doesn't
+ (yet) know about "skipping" data. */
+ wgint start = skip ? 0 : startpos;
+ progress = progress_create (start, start + toread);
progress_interactive = progress_interactive_p (progress);
}
should be read. */
while (!exact || (sum_read < toread))
{
- int rdsize = exact ? MIN (toread - sum_read, dlbufsize) : dlbufsize;
+ int rdsize;
double tmout = opt.read_timeout;
+
+ if (chunked)
+ {
+ if (remaining_chunk_size == 0)
+ {
+ char *line = fd_read_line (fd);
+ char *endl;
+ if (line == NULL)
+ {
+ ret = -1;
+ break;
+ }
+ else if (out2 != NULL)
+ fwrite (line, 1, strlen (line), out2);
+
+ remaining_chunk_size = strtol (line, &endl, 16);
+ xfree (line);
+
+ if (remaining_chunk_size == 0)
+ {
+ ret = 0;
+ line = fd_read_line (fd);
+ if (line == NULL)
+ ret = -1;
+ else
+ {
+ if (out2 != NULL)
+ fwrite (line, 1, strlen (line), out2);
+ xfree (line);
+ }
+ break;
+ }
+ }
+
+ rdsize = MIN (remaining_chunk_size, dlbufsize);
+ }
+ else
+ rdsize = exact ? MIN (toread - sum_read, dlbufsize) : dlbufsize;
+
if (progress_interactive)
- {
- /* For interactive progress gauges, always specify a ~1s
- timeout, so that the gauge can be updated regularly even
- when the data arrives very slowly or stalls. */
- tmout = 0.95;
- if (opt.read_timeout)
- {
- double waittm;
- waittm = (ptimer_read (timer) - last_successful_read_tm) / 1000;
- if (waittm + tmout > opt.read_timeout)
- {
- /* Don't let total idle time exceed read timeout. */
- tmout = opt.read_timeout - waittm;
- if (tmout < 0)
- {
- /* We've already exceeded the timeout. */
- ret = -1, errno = ETIMEDOUT;
- break;
- }
- }
- }
- }
+ {
+ /* For interactive progress gauges, always specify a ~1s
+ timeout, so that the gauge can be updated regularly even
+ when the data arrives very slowly or stalls. */
+ tmout = 0.95;
+ if (opt.read_timeout)
+ {
+ double waittm;
+ waittm = ptimer_read (timer) - last_successful_read_tm;
+ if (waittm + tmout > opt.read_timeout)
+ {
+ /* Don't let total idle time exceed read timeout. */
+ tmout = opt.read_timeout - waittm;
+ if (tmout < 0)
+ {
+ /* We've already exceeded the timeout. */
+ ret = -1, errno = ETIMEDOUT;
+ break;
+ }
+ }
+ }
+ }
ret = fd_read (fd, dlbuf, rdsize, tmout);
if (progress_interactive && ret < 0 && errno == ETIMEDOUT)
- ret = 0; /* interactive timeout, handled above */
+ ret = 0; /* interactive timeout, handled above */
else if (ret <= 0)
- break; /* EOF or read error */
+ break; /* EOF or read error */
- if (progress || opt.limit_rate)
- {
- ptimer_measure (timer);
- if (ret > 0)
- last_successful_read_tm = ptimer_read (timer);
- }
+ if (progress || opt.limit_rate || elapsed)
+ {
+ ptimer_measure (timer);
+ if (ret > 0)
+ last_successful_read_tm = ptimer_read (timer);
+ }
if (ret > 0)
- {
- sum_read += ret;
- if (!write_data (out, dlbuf, ret, &skip, &sum_written))
- {
- ret = -2;
- goto out;
- }
- }
+ {
+ sum_read += ret;
+ int write_res = write_data (out, out2, dlbuf, ret, &skip, &sum_written);
+ if (write_res < 0)
+ {
+ ret = (write_res == -3) ? -3 : -2;
+ goto out;
+ }
+ if (chunked)
+ {
+ remaining_chunk_size -= ret;
+ if (remaining_chunk_size == 0)
+ {
+ char *line = fd_read_line (fd);
+ if (line == NULL)
+ {
+ ret = -1;
+ break;
+ }
+ else
+ {
+ if (out2 != NULL)
+ fwrite (line, 1, strlen (line), out2);
+ xfree (line);
+ }
+ }
+ }
+ }
if (opt.limit_rate)
- limit_bandwidth (ret, timer);
+ limit_bandwidth (ret, timer);
if (progress)
- progress_update (progress, ret, ptimer_read (timer));
+ progress_update (progress, ret, ptimer_read (timer));
#ifdef WINDOWS
if (toread > 0 && !opt.quiet)
- ws_percenttitle (100.0 *
- (startpos + sum_read) / (startpos + toread));
+ ws_percenttitle (100.0 *
+ (startpos + sum_read) / (startpos + toread));
#endif
}
if (ret < -1)
if (qtywritten)
*qtywritten += sum_written;
+ free (dlbuf);
+
return ret;
}
\f
{
long bufsize = sizehint;
char *hunk = xmalloc (bufsize);
- int tail = 0; /* tail position in HUNK */
+ int tail = 0; /* tail position in HUNK */
- assert (maxsize >= bufsize);
+ assert (!maxsize || maxsize >= bufsize);
while (1)
{
pklen = fd_peek (fd, hunk + tail, bufsize - 1 - tail, -1);
if (pklen < 0)
- {
- xfree (hunk);
- return NULL;
- }
+ {
+ xfree (hunk);
+ return NULL;
+ }
end = terminator (hunk, hunk + tail, pklen);
if (end)
- {
- /* The data contains the terminator: we'll drain the data up
- to the end of the terminator. */
- remain = end - (hunk + tail);
- assert (remain >= 0);
- if (remain == 0)
- {
- /* No more data needs to be read. */
- hunk[tail] = '\0';
- return hunk;
- }
- if (bufsize - 1 < tail + remain)
- {
- bufsize = tail + remain + 1;
- hunk = xrealloc (hunk, bufsize);
- }
- }
+ {
+ /* The data contains the terminator: we'll drain the data up
+ to the end of the terminator. */
+ remain = end - (hunk + tail);
+ assert (remain >= 0);
+ if (remain == 0)
+ {
+ /* No more data needs to be read. */
+ hunk[tail] = '\0';
+ return hunk;
+ }
+ if (bufsize - 1 < tail + remain)
+ {
+ bufsize = tail + remain + 1;
+ hunk = xrealloc (hunk, bufsize);
+ }
+ }
else
- /* No terminator: simply read the data we know is (or should
- be) available. */
- remain = pklen;
+ /* No terminator: simply read the data we know is (or should
+ be) available. */
+ remain = pklen;
/* Now, read the data. Note that we make no assumptions about
- how much data we'll get. (Some TCP stacks are notorious for
- read returning less data than the previous MSG_PEEK.) */
+ how much data we'll get. (Some TCP stacks are notorious for
+ read returning less data than the previous MSG_PEEK.) */
rdlen = fd_read (fd, hunk + tail, remain, 0);
if (rdlen < 0)
- {
- xfree_null (hunk);
- return NULL;
- }
+ {
+ xfree_null (hunk);
+ return NULL;
+ }
tail += rdlen;
hunk[tail] = '\0';
if (rdlen == 0)
- {
- if (tail == 0)
- {
- /* EOF without anything having been read */
- xfree (hunk);
- errno = 0;
- return NULL;
- }
- else
- /* EOF seen: return the data we've read. */
- return hunk;
- }
+ {
+ if (tail == 0)
+ {
+ /* EOF without anything having been read */
+ xfree (hunk);
+ errno = 0;
+ return NULL;
+ }
+ else
+ /* EOF seen: return the data we've read. */
+ return hunk;
+ }
if (end && rdlen == remain)
- /* The terminator was seen and the remaining data drained --
- we got what we came for. */
- return hunk;
+ /* The terminator was seen and the remaining data drained --
+ we got what we came for. */
+ return hunk;
/* Keep looping until all the data arrives. */
if (tail == bufsize - 1)
- {
- /* Double the buffer size, but refuse to allocate more than
- MAXSIZE bytes. */
- if (maxsize && bufsize >= maxsize)
- {
- xfree (hunk);
- errno = ENOMEM;
- return NULL;
- }
- bufsize <<= 1;
- if (maxsize && bufsize > maxsize)
- bufsize = maxsize;
- hunk = xrealloc (hunk, bufsize);
- }
+ {
+ /* Double the buffer size, but refuse to allocate more than
+ MAXSIZE bytes. */
+ if (maxsize && bufsize >= maxsize)
+ {
+ xfree (hunk);
+ errno = ENOMEM;
+ return NULL;
+ }
+ bufsize <<= 1;
+ if (maxsize && bufsize > maxsize)
+ bufsize = maxsize;
+ hunk = xrealloc (hunk, bufsize);
+ }
}
}
the units appropriate for the download speed. */
const char *
-retr_rate (wgint bytes, double msecs)
+retr_rate (wgint bytes, double secs)
{
static char res[20];
static const char *rate_names[] = {"B/s", "KB/s", "MB/s", "GB/s" };
+ static const char *rate_names_bits[] = {"b/s", "Kb/s", "Mb/s", "Gb/s" };
int units;
- double dlrate = calc_rate (bytes, msecs, &units);
+ double dlrate = calc_rate (bytes, secs, &units);
/* Use more digits for smaller numbers (regardless of unit used),
e.g. "1022", "247", "12.5", "2.38". */
sprintf (res, "%.*f %s",
- dlrate >= 99.95 ? 0 : dlrate >= 9.995 ? 1 : 2,
- dlrate, rate_names[units]);
+ dlrate >= 99.95 ? 0 : dlrate >= 9.995 ? 1 : 2,
+ dlrate, !opt.report_bps ? rate_names[units]: rate_names_bits[units]);
return res;
}
UNITS is zero for B/s, one for KB/s, two for MB/s, and three for
GB/s. */
+
double
-calc_rate (wgint bytes, double msecs, int *units)
+calc_rate (wgint bytes, double secs, int *units)
{
double dlrate;
+ double bibyte = 1000.0;
+
+ if (!opt.report_bps)
+ bibyte = 1024.0;
- assert (msecs >= 0);
+
+ assert (secs >= 0);
assert (bytes >= 0);
- if (msecs == 0)
+ if (secs == 0)
/* If elapsed time is exactly zero, it means we're under the
resolution of the timer. This can easily happen on systems
that use time() for the timer. Since the interval lies between
0 and the timer's resolution, assume half the resolution. */
- msecs = ptimer_resolution () / 2.0;
+ secs = ptimer_resolution () / 2.0;
- dlrate = 1000.0 * bytes / msecs;
- if (dlrate < 1024.0)
+ dlrate = convert_to_bits (bytes) / secs;
+ if (dlrate < bibyte)
*units = 0;
- else if (dlrate < 1024.0 * 1024.0)
- *units = 1, dlrate /= 1024.0;
- else if (dlrate < 1024.0 * 1024.0 * 1024.0)
- *units = 2, dlrate /= (1024.0 * 1024.0);
+ else if (dlrate < (bibyte * bibyte))
+ *units = 1, dlrate /= bibyte;
+ else if (dlrate < (bibyte * bibyte * bibyte))
+ *units = 2, dlrate /= (bibyte * bibyte);
+
else
/* Maybe someone will need this, one day. */
- *units = 3, dlrate /= (1024.0 * 1024.0 * 1024.0);
+ *units = 3, dlrate /= (bibyte * bibyte * bibyte);
return dlrate;
}
\f
-/* Maximum number of allowed redirections. 20 was chosen as a
- "reasonable" value, which is low enough to not cause havoc, yet
- high enough to guarantee that normal retrievals will not be hurt by
- the check. */
-
-#define MAX_REDIRECTIONS 20
-
-#define SUSPEND_POST_DATA do { \
- post_data_suspended = true; \
- saved_post_data = opt.post_data; \
- saved_post_file_name = opt.post_file_name; \
- opt.post_data = NULL; \
- opt.post_file_name = NULL; \
+
+#define SUSPEND_METHOD do { \
+ method_suspended = true; \
+ saved_body_data = opt.body_data; \
+ saved_body_file_name = opt.body_file; \
+ saved_method = opt.method; \
+ opt.body_data = NULL; \
+ opt.body_file = NULL; \
+ opt.method = NULL; \
} while (0)
-#define RESTORE_POST_DATA do { \
- if (post_data_suspended) \
- { \
- opt.post_data = saved_post_data; \
- opt.post_file_name = saved_post_file_name; \
- post_data_suspended = false; \
- } \
+#define RESTORE_METHOD do { \
+ if (method_suspended) \
+ { \
+ opt.body_data = saved_body_data; \
+ opt.body_file = saved_body_file_name; \
+ opt.method = saved_method; \
+ method_suspended = false; \
+ } \
} while (0)
static char *getproxy (struct url *);
multiple points. */
uerr_t
-retrieve_url (const char *origurl, char **file, char **newloc,
- const char *refurl, int *dt)
+retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
+ char **newloc, const char *refurl, int *dt, bool recursive,
+ struct iri *iri, bool register_status)
{
uerr_t result;
char *url;
bool location_changed;
+ bool iri_fallbacked = 0;
int dummy;
char *mynewloc, *proxy;
- struct url *u, *proxy_url;
- int up_error_code; /* url parse error code */
+ struct url *u = orig_parsed, *proxy_url;
+ int up_error_code; /* url parse error code */
char *local_file;
int redirection_count = 0;
- bool post_data_suspended = false;
- char *saved_post_data = NULL;
- char *saved_post_file_name = NULL;
+ bool method_suspended = false;
+ char *saved_body_data = NULL;
+ char *saved_method = NULL;
+ char *saved_body_file_name = NULL;
/* If dt is NULL, use local storage. */
if (!dt)
if (file)
*file = NULL;
- u = url_parse (url, &up_error_code);
- if (!u)
- {
- logprintf (LOG_NOTQUIET, "%s: %s.\n", url, url_error (up_error_code));
- xfree (url);
- return URLERROR;
- }
-
if (!refurl)
refurl = opt.referer;
redirected:
+ /* (also for IRI fallbacking) */
result = NOCONERROR;
mynewloc = NULL;
proxy = getproxy (u);
if (proxy)
{
+ struct iri *pi = iri_new ();
+ set_uri_encoding (pi, opt.locale, true);
+ pi->utf8_encode = false;
+
/* Parse the proxy URL. */
- proxy_url = url_parse (proxy, &up_error_code);
+ proxy_url = url_parse (proxy, &up_error_code, NULL, true);
if (!proxy_url)
- {
- logprintf (LOG_NOTQUIET, _("Error parsing proxy URL %s: %s.\n"),
- proxy, url_error (up_error_code));
- xfree (url);
- RESTORE_POST_DATA;
- return PROXERR;
- }
+ {
+ char *error = url_error (proxy, up_error_code);
+ logprintf (LOG_NOTQUIET, _("Error parsing proxy URL %s: %s.\n"),
+ proxy, error);
+ xfree (url);
+ xfree (error);
+ RESTORE_METHOD;
+ result = PROXERR;
+ goto bail;
+ }
if (proxy_url->scheme != SCHEME_HTTP && proxy_url->scheme != u->scheme)
- {
- logprintf (LOG_NOTQUIET, _("Error in proxy URL %s: Must be HTTP.\n"), proxy);
- url_free (proxy_url);
- xfree (url);
- RESTORE_POST_DATA;
- return PROXERR;
- }
+ {
+ logprintf (LOG_NOTQUIET, _("Error in proxy URL %s: Must be HTTP.\n"), proxy);
+ url_free (proxy_url);
+ xfree (url);
+ RESTORE_METHOD;
+ result = PROXERR;
+ goto bail;
+ }
}
if (u->scheme == SCHEME_HTTP
#endif
|| (proxy_url && proxy_url->scheme == SCHEME_HTTP))
{
- result = http_loop (u, &mynewloc, &local_file, refurl, dt, proxy_url);
+ result = http_loop (u, orig_parsed, &mynewloc, &local_file, refurl, dt,
+ proxy_url, iri);
}
else if (u->scheme == SCHEME_FTP)
{
/* If this is a redirection, temporarily turn off opt.ftp_glob
- and opt.recursive, both being undesirable when following
- redirects. */
- bool oldrec = opt.recursive, oldglob = opt.ftp_glob;
+ and opt.recursive, both being undesirable when following
+ redirects. */
+ bool oldrec = recursive, glob = opt.ftp_glob;
if (redirection_count)
- opt.recursive = opt.ftp_glob = false;
+ oldrec = glob = false;
- result = ftp_loop (u, dt, proxy_url);
- opt.recursive = oldrec;
- opt.ftp_glob = oldglob;
+ result = ftp_loop (u, &local_file, dt, proxy_url, recursive, glob);
+ recursive = oldrec;
/* There is a possibility of having HTTP being redirected to
- FTP. In these cases we must decide whether the text is HTML
- according to the suffix. The HTML suffixes are `.html',
- `.htm' and a few others, case-insensitive. */
+ FTP. In these cases we must decide whether the text is HTML
+ according to the suffix. The HTML suffixes are `.html',
+ `.htm' and a few others, case-insensitive. */
if (redirection_count && local_file && u->scheme == SCHEME_FTP)
- {
- if (has_html_suffix_p (local_file))
- *dt |= TEXTHTML;
- }
+ {
+ if (has_html_suffix_p (local_file))
+ *dt |= TEXTHTML;
+ }
}
if (proxy_url)
proxy_url = NULL;
}
- location_changed = (result == NEWLOCATION);
+ location_changed = (result == NEWLOCATION || result == NEWLOCATION_KEEP_POST);
if (location_changed)
{
char *construced_newloc;
assert (mynewloc != NULL);
if (local_file)
- xfree (local_file);
+ xfree (local_file);
/* The HTTP specs only allow absolute URLs to appear in
- redirects, but a ton of boneheaded webservers and CGIs out
- there break the rules and use relative URLs, and popular
- browsers are lenient about this, so wget should be too. */
+ redirects, but a ton of boneheaded webservers and CGIs out
+ there break the rules and use relative URLs, and popular
+ browsers are lenient about this, so wget should be too. */
construced_newloc = uri_merge (url, mynewloc);
xfree (mynewloc);
mynewloc = construced_newloc;
+ /* Reset UTF-8 encoding state, keep the URI encoding and reset
+ the content encoding. */
+ iri->utf8_encode = opt.enable_iri;
+ set_content_encoding (iri, NULL);
+ xfree_null (iri->orig_url);
+ iri->orig_url = NULL;
+
/* Now, see if this new location makes sense. */
- newloc_parsed = url_parse (mynewloc, &up_error_code);
+ newloc_parsed = url_parse (mynewloc, &up_error_code, iri, true);
if (!newloc_parsed)
- {
- logprintf (LOG_NOTQUIET, "%s: %s.\n", escnonprint_uri (mynewloc),
- url_error (up_error_code));
- url_free (u);
- xfree (url);
- xfree (mynewloc);
- RESTORE_POST_DATA;
- return result;
- }
+ {
+ char *error = url_error (mynewloc, up_error_code);
+ logprintf (LOG_NOTQUIET, "%s: %s.\n", escnonprint_uri (mynewloc),
+ error);
+ if (orig_parsed != u)
+ {
+ url_free (u);
+ }
+ xfree (url);
+ xfree (mynewloc);
+ xfree (error);
+ RESTORE_METHOD;
+ goto bail;
+ }
/* Now mynewloc will become newloc_parsed->url, because if the
Location contained relative paths like .././something, we
mynewloc = xstrdup (newloc_parsed->url);
/* Check for max. number of redirections. */
- if (++redirection_count > MAX_REDIRECTIONS)
- {
- logprintf (LOG_NOTQUIET, _("%d redirections exceeded.\n"),
- MAX_REDIRECTIONS);
- url_free (newloc_parsed);
- url_free (u);
- xfree (url);
- xfree (mynewloc);
- RESTORE_POST_DATA;
- return WRONGCODE;
- }
+ if (++redirection_count > opt.max_redirect)
+ {
+ logprintf (LOG_NOTQUIET, _("%d redirections exceeded.\n"),
+ opt.max_redirect);
+ url_free (newloc_parsed);
+ if (orig_parsed != u)
+ {
+ url_free (u);
+ }
+ xfree (url);
+ xfree (mynewloc);
+ RESTORE_METHOD;
+ result = WRONGCODE;
+ goto bail;
+ }
xfree (url);
url = mynewloc;
- url_free (u);
+ if (orig_parsed != u)
+ {
+ url_free (u);
+ }
u = newloc_parsed;
- /* If we're being redirected from POST, we don't want to POST
- again. Many requests answer POST with a redirection to an
- index page; that redirection is clearly a GET. We "suspend"
- POST data for the duration of the redirections, and restore
- it when we're done. */
- if (!post_data_suspended)
- SUSPEND_POST_DATA;
+ /* If we're being redirected from POST, and we received a
+ redirect code different than 307, we don't want to POST
+ again. Many requests answer POST with a redirection to an
+ index page; that redirection is clearly a GET. We "suspend"
+ POST data for the duration of the redirections, and restore
+ it when we're done.
+
+ RFC2616 HTTP/1.1 introduces code 307 Temporary Redirect
+ specifically to preserve the method of the request.
+ */
+ if (result != NEWLOCATION_KEEP_POST && !method_suspended)
+ SUSPEND_METHOD;
goto redirected;
}
- if (local_file)
+ /* Try to not encode in UTF-8 if fetching failed */
+ if (!(*dt & RETROKF) && iri->utf8_encode)
+ {
+ iri->utf8_encode = false;
+ if (orig_parsed != u)
+ {
+ url_free (u);
+ }
+ u = url_parse (origurl, NULL, iri, true);
+ if (u)
+ {
+ DEBUGP (("[IRI fallbacking to non-utf8 for %s\n", quote (url)));
+ url = xstrdup (u->url);
+ iri_fallbacked = 1;
+ goto redirected;
+ }
+ else
+ DEBUGP (("[Couldn't fallback to non-utf8 for %s\n", quote (url)));
+ }
+
+ if (local_file && u && *dt & RETROKF)
{
- if (*dt & RETROKF)
- {
- register_download (u->url, local_file);
- if (redirection_count && 0 != strcmp (origurl, u->url))
- register_redirection (origurl, u->url);
- if (*dt & TEXTHTML)
- register_html (u->url, local_file);
- }
+ register_download (u->url, local_file);
+
+ if (!opt.spider && redirection_count && 0 != strcmp (origurl, u->url))
+ register_redirection (origurl, u->url);
+
+ if (*dt & TEXTHTML)
+ register_html (local_file);
+
+ if (*dt & TEXTCSS)
+ register_css (local_file);
}
if (file)
else
xfree_null (local_file);
- url_free (u);
+ if (orig_parsed != u)
+ {
+ url_free (u);
+ }
- if (redirection_count)
+ if (redirection_count || iri_fallbacked)
{
if (newloc)
- *newloc = url;
+ *newloc = url;
else
- xfree (url);
+ xfree (url);
}
else
{
if (newloc)
- *newloc = NULL;
+ *newloc = NULL;
xfree (url);
}
- RESTORE_POST_DATA;
+ RESTORE_METHOD;
+bail:
+ if (register_status)
+ inform_exit_status (result);
return result;
}
{
uerr_t status;
struct urlpos *url_list, *cur_url;
+ struct iri *iri = iri_new();
+
+ char *input_file, *url_file = NULL;
+ const char *url = file;
- url_list = (html ? get_urls_html (file, NULL, NULL)
- : get_urls_file (file));
status = RETROK; /* Suppose everything is OK. */
*count = 0; /* Reset the URL count. */
+ /* sXXXav : Assume filename and links in the file are in the locale */
+ set_uri_encoding (iri, opt.locale, true);
+ set_content_encoding (iri, opt.locale);
+
+ if (url_valid_scheme (url))
+ {
+ int dt,url_err;
+ uerr_t status;
+ struct url *url_parsed = url_parse (url, &url_err, iri, true);
+ if (!url_parsed)
+ {
+ char *error = url_error (url, url_err);
+ logprintf (LOG_NOTQUIET, "%s: %s.\n", url, error);
+ xfree (error);
+ return URLERROR;
+ }
+
+ if (!opt.base_href)
+ opt.base_href = xstrdup (url);
+
+ status = retrieve_url (url_parsed, url, &url_file, NULL, NULL, &dt,
+ false, iri, true);
+ url_free (url_parsed);
+
+ if (!url_file || (status != RETROK))
+ return status;
+
+ if (dt & TEXTHTML)
+ html = true;
+
+ /* If we have a found a content encoding, use it.
+ * ( == is okay, because we're checking for identical object) */
+ if (iri->content_encoding != opt.locale)
+ set_uri_encoding (iri, iri->content_encoding, false);
+
+ /* Reset UTF-8 encode status */
+ iri->utf8_encode = opt.enable_iri;
+ xfree_null (iri->orig_url);
+ iri->orig_url = NULL;
+
+ input_file = url_file;
+ }
+ else
+ input_file = (char *) file;
+
+ url_list = (html ? get_urls_html (input_file, NULL, NULL, iri)
+ : get_urls_file (input_file));
+
+ xfree_null (url_file);
+
for (cur_url = url_list; cur_url; cur_url = cur_url->next, ++*count)
{
char *filename = NULL, *new_file = NULL;
int dt;
+ struct iri *tmpiri = iri_dup (iri);
+ struct url *parsed_url = NULL;
if (cur_url->ignore_when_downloading)
- continue;
+ continue;
if (opt.quota && total_downloaded_bytes > opt.quota)
- {
- status = QUOTEXC;
- break;
- }
+ {
+ status = QUOTEXC;
+ break;
+ }
+
+ parsed_url = url_parse (cur_url->url->url, NULL, tmpiri, true);
+
if ((opt.recursive || opt.page_requisites)
- && cur_url->url->scheme != SCHEME_FTP)
- status = retrieve_tree (cur_url->url->url);
+ && (cur_url->url->scheme != SCHEME_FTP || getproxy (cur_url->url)))
+ {
+ int old_follow_ftp = opt.follow_ftp;
+
+ /* Turn opt.follow_ftp on in case of recursive FTP retrieval */
+ if (cur_url->url->scheme == SCHEME_FTP)
+ opt.follow_ftp = 1;
+
+ status = retrieve_tree (parsed_url ? parsed_url : cur_url->url,
+ tmpiri);
+
+ opt.follow_ftp = old_follow_ftp;
+ }
else
- status = retrieve_url (cur_url->url->url, &filename, &new_file, NULL, &dt);
+ status = retrieve_url (parsed_url ? parsed_url : cur_url->url,
+ cur_url->url->url, &filename,
+ &new_file, NULL, &dt, opt.recursive, tmpiri,
+ true);
+
+ if (parsed_url)
+ url_free (parsed_url);
if (filename && opt.delete_after && file_exists_p (filename))
- {
- DEBUGP (("\
+ {
+ DEBUGP (("\
Removing file due to --delete-after in retrieve_from_file():\n"));
- logprintf (LOG_VERBOSE, _("Removing %s.\n"), filename);
- if (unlink (filename))
- logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
- dt &= ~RETROKF;
- }
+ logprintf (LOG_VERBOSE, _("Removing %s.\n"), filename);
+ if (unlink (filename))
+ logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
+ dt &= ~RETROKF;
+ }
xfree_null (new_file);
xfree_null (filename);
+ iri_free (tmpiri);
}
/* Free the linked list of URL-s. */
free_urlpos (url_list);
+ iri_free (iri);
+
return status;
}
if (opt.waitretry && count > 1)
{
/* If opt.waitretry is specified and this is a retry, wait for
- COUNT-1 number of seconds, or for opt.waitretry seconds. */
+ COUNT-1 number of seconds, or for opt.waitretry seconds. */
if (count <= opt.waitretry)
- xsleep (count - 1);
+ xsleep (count - 1);
else
- xsleep (opt.waitretry);
+ xsleep (opt.waitretry);
}
else if (opt.wait)
{
if (!opt.random_wait || count > 1)
- /* If random-wait is not specified, or if we are sleeping
- between retries of the same download, sleep the fixed
- interval. */
- xsleep (opt.wait);
+ /* If random-wait is not specified, or if we are sleeping
+ between retries of the same download, sleep the fixed
+ interval. */
+ xsleep (opt.wait);
else
- {
- /* Sleep a random amount of time averaging in opt.wait
- seconds. The sleeping amount ranges from 0 to
- opt.wait*2, inclusive. */
- double waitsecs = 2 * opt.wait * random_float ();
- DEBUGP (("sleep_between_retrievals: avg=%f,sleep=%f\n",
- opt.wait, waitsecs));
- xsleep (waitsecs);
- }
+ {
+ /* Sleep a random amount of time averaging in opt.wait
+ seconds. The sleeping amount ranges from 0.5*opt.wait to
+ 1.5*opt.wait. */
+ double waitsecs = (0.5 + random_float ()) * opt.wait;
+ DEBUGP (("sleep_between_retrievals: avg=%f,sleep=%f\n",
+ opt.wait, waitsecs));
+ xsleep (waitsecs);
+ }
}
}
{
struct urlpos *next = l->next;
if (l->url)
- url_free (l->url);
+ url_free (l->url);
xfree_null (l->local_name);
xfree (l);
l = next;
void
rotate_backups(const char *fname)
{
- int maxlen = strlen (fname) + 1 + numdigit (opt.backups) + 1;
+#ifdef __VMS
+# define SEP "_"
+# define AVS ";*" /* All-version suffix. */
+# define AVSL (sizeof (AVS) - 1)
+#else
+# define SEP "."
+# define AVSL 0
+#endif
+
+ int maxlen = strlen (fname) + sizeof (SEP) + numdigit (opt.backups) + AVSL;
char *from = (char *)alloca (maxlen);
char *to = (char *)alloca (maxlen);
struct_stat sb;
for (i = opt.backups; i > 1; i--)
{
- sprintf (from, "%s.%d", fname, i - 1);
- sprintf (to, "%s.%d", fname, i);
+#ifdef VMS
+ /* Delete (all versions of) any existing max-suffix file, to avoid
+ * creating multiple versions of it. (On VMS, rename() will
+ * create a new version of an existing destination file, not
+ * destroy/overwrite it.)
+ */
+ if (i == opt.backups)
+ {
+ sprintf (to, "%s%s%d%s", fname, SEP, i, AVS);
+ delete (to);
+ }
+#endif
+ sprintf (to, "%s%s%d", fname, SEP, i);
+ sprintf (from, "%s%s%d", fname, SEP, i - 1);
rename (from, to);
}
- sprintf (to, "%s.%d", fname, 1);
+ sprintf (to, "%s%s%d", fname, SEP, 1);
rename(fname, to);
}
if (!opt.use_proxy)
return NULL;
- if (!no_proxy_match (u->host, (const char **)opt.no_proxy))
+ if (no_proxy_match (u->host, (const char **)opt.no_proxy))
return NULL;
switch (u->scheme)
return proxy;
}
+/* Returns true if URL would be downloaded through a proxy. */
+
+bool
+url_uses_proxy (struct url * u)
+{
+ bool ret;
+ if (!u)
+ return false;
+ ret = getproxy (u) != NULL;
+ return ret;
+}
+
/* Should a host be accessed through proxy, concerning no_proxy? */
static bool
no_proxy_match (const char *host, const char **no_proxy)
{
if (!no_proxy)
- return true;
+ return false;
+ else
+ return sufmatch (no_proxy, host);
+}
+
+/* Set the file parameter to point to the local file string. */
+void
+set_local_file (const char **file, const char *default_file)
+{
+ if (opt.output_document)
+ {
+ if (output_stream_regular)
+ *file = opt.output_document;
+ }
+ else
+ *file = default_file;
+}
+
+/* Return true for an input file's own URL, false otherwise. */
+bool
+input_file_url (const char *input_file)
+{
+ static bool first = true;
+
+ if (input_file
+ && url_has_scheme (input_file)
+ && first)
+ {
+ first = false;
+ return true;
+ }
else
- return !sufmatch (no_proxy, host);
+ return false;
}
projects / wget / blobdiff