/* File retrieval.
- Copyright (C) 1995, 1996, 1997, 1998, 2000, 2001 Free Software Foundation, Inc.
+ Copyright (C) 1996-2006 Free Software Foundation, Inc.
This file is part of GNU Wget.
GNU Wget is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or (at
+the Free Software Foundation; either version 3 of the License, or (at
your option) any later version.
GNU Wget is distributed in the hope that it will be useful,
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
-along with Wget; if not, write to the Free Software
-Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+along with Wget. If not, see <http://www.gnu.org/licenses/>.
In addition, as a special exception, the Free Software Foundation
gives permission to link the code of its release of Wget with the
#include <stdio.h>
#include <stdlib.h>
-#include <sys/types.h>
#ifdef HAVE_UNISTD_H
# include <unistd.h>
#endif /* HAVE_UNISTD_H */
#include <errno.h>
-#ifdef HAVE_STRING_H
-# include <string.h>
-#else
-# include <strings.h>
-#endif /* HAVE_STRING_H */
+#include <string.h>
#include <assert.h>
#include "wget.h"
#include "url.h"
#include "recur.h"
#include "ftp.h"
+#include "http.h"
#include "host.h"
#include "connect.h"
#include "hash.h"
#include "convert.h"
-
-#ifdef HAVE_SSL
-# include "gen_sslfunc.h" /* for ssl_iread */
-#endif
-
-#ifndef errno
-extern int errno;
-#endif
+#include "ptimer.h"
/* Total size of downloaded files. Used to enforce quota. */
-LARGE_INT total_downloaded_bytes;
+SUM_SIZE_INT total_downloaded_bytes;
+
+/* Total download time in seconds. */
+double total_download_time;
/* If non-NULL, the stream to which output should be written. This
stream is initialized when `-O' is used. */
/* Whether output_document is a regular file we can manipulate,
i.e. not `-' or a device file. */
-int output_stream_regular;
+bool output_stream_regular;
\f
static struct {
- long chunk_bytes;
+ wgint chunk_bytes;
double chunk_start;
double sleep_adjust;
} limit_data;
static void
limit_bandwidth_reset (void)
{
- limit_data.chunk_bytes = 0;
- limit_data.chunk_start = 0;
+ xzero (limit_data);
}
/* Limit the bandwidth by pausing the download for an amount of time.
is the timer that started at the beginning of download. */
static void
-limit_bandwidth (long bytes, struct wget_timer *timer)
+limit_bandwidth (wgint bytes, struct ptimer *timer)
{
- double delta_t = wtimer_read (timer) - limit_data.chunk_start;
+ double delta_t = ptimer_read (timer) - limit_data.chunk_start;
double expected;
limit_data.chunk_bytes += bytes;
/* Calculate the amount of time we expect downloading the chunk
should take. If in reality it took less time, sleep to
compensate for the difference. */
- expected = 1000.0 * limit_data.chunk_bytes / opt.limit_rate;
+ expected = (double) limit_data.chunk_bytes / opt.limit_rate;
if (expected > delta_t)
{
double slp = expected - delta_t + limit_data.sleep_adjust;
double t0, t1;
- if (slp < 200)
+ if (slp < 0.2)
{
- DEBUGP (("deferring a %.2f ms sleep (%ld/%.2f).\n",
- slp, limit_data.chunk_bytes, delta_t));
+ DEBUGP (("deferring a %.2f ms sleep (%s/%.2f).\n",
+ slp * 1000, number_to_static_string (limit_data.chunk_bytes),
+ delta_t));
return;
}
- DEBUGP (("\nsleeping %.2f ms for %ld bytes, adjust %.2f ms\n",
- slp, limit_data.chunk_bytes, limit_data.sleep_adjust));
+ DEBUGP (("\nsleeping %.2f ms for %s bytes, adjust %.2f ms\n",
+ slp * 1000, number_to_static_string (limit_data.chunk_bytes),
+ limit_data.sleep_adjust));
- t0 = wtimer_read (timer);
- xsleep (slp / 1000);
- wtimer_update (timer);
- t1 = wtimer_read (timer);
+ t0 = ptimer_read (timer);
+ xsleep (slp);
+ t1 = ptimer_measure (timer);
/* Due to scheduling, we probably slept slightly longer (or
shorter) than desired. Calculate the difference between the
desired and the actual sleep, and adjust the next sleep by
that amount. */
limit_data.sleep_adjust = slp - (t1 - t0);
+ /* If sleep_adjust is very large, it's likely due to suspension
+ and not clock inaccuracy. Don't enforce those. */
+ if (limit_data.sleep_adjust > 0.5)
+ limit_data.sleep_adjust = 0.5;
+ else if (limit_data.sleep_adjust < -0.5)
+ limit_data.sleep_adjust = -0.5;
}
limit_data.chunk_bytes = 0;
- limit_data.chunk_start = wtimer_read (timer);
+ limit_data.chunk_start = ptimer_read (timer);
}
#ifndef MIN
of data written. */
static int
-write_data (FILE *out, const char *buf, int bufsize, long *skip,
- long *written)
+write_data (FILE *out, const char *buf, int bufsize, wgint *skip,
+ wgint *written)
{
if (!out)
return 1;
is incremented by the amount of data read from the network. If
QTYWRITTEN is non-NULL, the value it points to is incremented by
the amount of data written to disk. The time it took to download
- the data (in milliseconds) is stored to ELAPSED.
+ the data is stored to ELAPSED.
The function exits and returns the amount of data read. In case of
error while reading data, -1 is returned. In case of error while
writing data, -2 is returned. */
int
-fd_read_body (int fd, FILE *out, long toread, long startpos,
- long *qtyread, long *qtywritten, double *elapsed, int flags)
+fd_read_body (int fd, FILE *out, wgint toread, wgint startpos,
+ wgint *qtyread, wgint *qtywritten, double *elapsed, int flags)
{
int ret = 0;
static char dlbuf[16384];
int dlbufsize = sizeof (dlbuf);
- struct wget_timer *timer = NULL;
+ struct ptimer *timer = NULL;
double last_successful_read_tm = 0;
/* The progress gauge, set according to the user preferences. */
continually update the display. When true, smaller timeout
values are used so that the gauge can update the display when
data arrives slowly. */
- int progress_interactive = 0;
+ bool progress_interactive = false;
- int exact = flags & rb_read_exactly;
- long skip = 0;
+ bool exact = !!(flags & rb_read_exactly);
+ wgint skip = 0;
/* How much data we've read/written. */
- long sum_read = 0;
- long sum_written = 0;
+ wgint sum_read = 0;
+ wgint sum_written = 0;
if (flags & rb_skip_startpos)
skip = startpos;
the timer. */
if (progress || opt.limit_rate || elapsed)
{
- timer = wtimer_new ();
+ timer = ptimer_new ();
last_successful_read_tm = 0;
}
if (opt.read_timeout)
{
double waittm;
- waittm = (wtimer_read (timer) - last_successful_read_tm) / 1000;
+ waittm = ptimer_read (timer) - last_successful_read_tm;
if (waittm + tmout > opt.read_timeout)
{
/* Don't let total idle time exceed read timeout. */
}
ret = fd_read (fd, dlbuf, rdsize, tmout);
- if (ret == 0 || (ret < 0 && errno != ETIMEDOUT))
- break; /* read error */
- else if (ret < 0)
- ret = 0; /* read timeout */
+ if (progress_interactive && ret < 0 && errno == ETIMEDOUT)
+ ret = 0; /* interactive timeout, handled above */
+ else if (ret <= 0)
+ break; /* EOF or read error */
if (progress || opt.limit_rate)
{
- wtimer_update (timer);
+ ptimer_measure (timer);
if (ret > 0)
- last_successful_read_tm = wtimer_read (timer);
+ last_successful_read_tm = ptimer_read (timer);
}
if (ret > 0)
limit_bandwidth (ret, timer);
if (progress)
- progress_update (progress, ret, wtimer_read (timer));
+ progress_update (progress, ret, ptimer_read (timer));
#ifdef WINDOWS
if (toread > 0 && !opt.quiet)
ws_percenttitle (100.0 *
out:
if (progress)
- progress_finish (progress, wtimer_read (timer));
+ progress_finish (progress, ptimer_read (timer));
if (elapsed)
- *elapsed = wtimer_read (timer);
+ *elapsed = ptimer_read (timer);
if (timer)
- wtimer_delete (timer);
+ ptimer_destroy (timer);
if (qtyread)
*qtyread += sum_read;
return ret;
}
\f
-/* Read a hunk of data from FD, up until a terminator. The terminator
- is whatever the TERMINATOR function determines it to be; for
- example, it can be a line of data, or the head of an HTTP response.
- The function returns the data read allocated with malloc.
-
- In case of error, NULL is returned. In case of EOF and no data
- read, NULL is returned and errno set to 0. In case of EOF with
- data having been read, the data is returned, but it will
- (obviously) not contain the terminator.
+/* Read a hunk of data from FD, up until a terminator. The hunk is
+ limited by whatever the TERMINATOR callback chooses as its
+ terminator. For example, if terminator stops at newline, the hunk
+ will consist of a line of data; if terminator stops at two
+ newlines, it can be used to read the head of an HTTP response.
+ Upon determining the boundary, the function returns the data (up to
+ the terminator) in malloc-allocated storage.
+
+ In case of read error, NULL is returned. In case of EOF and no
+ data read, NULL is returned and errno set to 0. In case of having
+ read some data, but encountering EOF before seeing the terminator,
+ the data that has been read is returned, but it will (obviously)
+ not contain the terminator.
+
+ The TERMINATOR function is called with three arguments: the
+ beginning of the data read so far, the beginning of the current
+ block of peeked-at data, and the length of the current block.
+ Depending on its needs, the function is free to choose whether to
+ analyze all data or just the newly arrived data. If TERMINATOR
+ returns NULL, it means that the terminator has not been seen.
+ Otherwise it should return a pointer to the charactre immediately
+ following the terminator.
The idea is to be able to read a line of input, or otherwise a hunk
of text, such as the head of an HTTP request, without crossing the
boundary, so that the next call to fd_read etc. reads the data
after the hunk. To achieve that, this function does the following:
- 1. Peek at available data.
+ 1. Peek at incoming data.
2. Determine whether the peeked data, along with the previously
read data, includes the terminator.
a read. If the read returns a different amount of data, the
process is retried until all data arrives safely.
- BUFSIZE is the size of the initial buffer expected to read all the
- data in the typical case.
+ SIZEHINT is the buffer size sufficient to hold all the data in the
+ typical case (it is used as the initial buffer size). MAXSIZE is
+ the maximum amount of memory this function is allowed to allocate,
+ or 0 if no upper limit is to be enforced.
This function should be used as a building block for other
functions -- see fd_read_line as a simple example. */
char *
-fd_read_hunk (int fd, hunk_terminator_t hunk_terminator, int bufsize)
+fd_read_hunk (int fd, hunk_terminator_t terminator, long sizehint, long maxsize)
{
+ long bufsize = sizehint;
char *hunk = xmalloc (bufsize);
int tail = 0; /* tail position in HUNK */
+ assert (maxsize >= bufsize);
+
while (1)
{
const char *end;
xfree (hunk);
return NULL;
}
- end = hunk_terminator (hunk, tail, pklen);
+ end = terminator (hunk, hunk + tail, pklen);
if (end)
{
/* The data contains the terminator: we'll drain the data up
to the end of the terminator. */
remain = end - (hunk + tail);
+ assert (remain >= 0);
if (remain == 0)
{
/* No more data needs to be read. */
if (tail == bufsize - 1)
{
+ /* Double the buffer size, but refuse to allocate more than
+ MAXSIZE bytes. */
+ if (maxsize && bufsize >= maxsize)
+ {
+ xfree (hunk);
+ errno = ENOMEM;
+ return NULL;
+ }
bufsize <<= 1;
+ if (maxsize && bufsize > maxsize)
+ bufsize = maxsize;
hunk = xrealloc (hunk, bufsize);
}
}
}
static const char *
-line_terminator (const char *hunk, int oldlen, int peeklen)
+line_terminator (const char *start, const char *peeked, int peeklen)
{
- const char *p = memchr (hunk + oldlen, '\n', peeklen);
+ const char *p = memchr (peeked, '\n', peeklen);
if (p)
- /* p+1 because we want the line to include '\n' */
+ /* p+1 because the line must include '\n' */
return p + 1;
return NULL;
}
+/* The maximum size of the single line we agree to accept. This is
+ not meant to impose an arbitrary limit, but to protect the user
+ from Wget slurping up available memory upon encountering malicious
+ or buggy server output. Define it to 0 to remove the limit. */
+#define FD_READ_LINE_MAX 4096
+
/* Read one line from FD and return it. The line is allocated using
- malloc.
+ malloc, but is never larger than FD_READ_LINE_MAX.
If an error occurs, or if no data can be read, NULL is returned.
In the former case errno indicates the error condition, and in the
char *
fd_read_line (int fd)
{
- return fd_read_hunk (fd, line_terminator, 128);
+ return fd_read_hunk (fd, line_terminator, 128, FD_READ_LINE_MAX);
}
\f
-/* Return a printed representation of the download rate, as
- appropriate for the speed. If PAD is non-zero, strings will be
- padded to the width of 7 characters (xxxx.xx). */
-char *
-retr_rate (long bytes, double msecs, int pad)
+/* Return a printed representation of the download rate, along with
+ the units appropriate for the download speed. */
+
+const char *
+retr_rate (wgint bytes, double secs)
{
static char res[20];
static const char *rate_names[] = {"B/s", "KB/s", "MB/s", "GB/s" };
- int units = 0;
+ int units;
- double dlrate = calc_rate (bytes, msecs, &units);
- sprintf (res, pad ? "%7.2f %s" : "%.2f %s", dlrate, rate_names[units]);
+ double dlrate = calc_rate (bytes, secs, &units);
+ /* Use more digits for smaller numbers (regardless of unit used),
+ e.g. "1022", "247", "12.5", "2.38". */
+ sprintf (res, "%.*f %s",
+ dlrate >= 99.95 ? 0 : dlrate >= 9.995 ? 1 : 2,
+ dlrate, rate_names[units]);
return res;
}
UNITS is zero for B/s, one for KB/s, two for MB/s, and three for
GB/s. */
+
double
-calc_rate (long bytes, double msecs, int *units)
+calc_rate (wgint bytes, double secs, int *units)
{
double dlrate;
- assert (msecs >= 0);
+ assert (secs >= 0);
assert (bytes >= 0);
- if (msecs == 0)
+ if (secs == 0)
/* If elapsed time is exactly zero, it means we're under the
- granularity of the timer. This often happens on systems that
- use time() for the timer. */
- msecs = wtimer_granularity ();
+ resolution of the timer. This can easily happen on systems
+ that use time() for the timer. Since the interval lies between
+ 0 and the timer's resolution, assume half the resolution. */
+ secs = ptimer_resolution () / 2.0;
- dlrate = (double)1000 * bytes / msecs;
+ dlrate = bytes / secs;
if (dlrate < 1024.0)
*units = 0;
else if (dlrate < 1024.0 * 1024.0)
#define MAX_REDIRECTIONS 20
#define SUSPEND_POST_DATA do { \
- post_data_suspended = 1; \
+ post_data_suspended = true; \
saved_post_data = opt.post_data; \
saved_post_file_name = opt.post_file_name; \
opt.post_data = NULL; \
{ \
opt.post_data = saved_post_data; \
opt.post_file_name = saved_post_file_name; \
- post_data_suspended = 0; \
+ post_data_suspended = false; \
} \
} while (0)
-static char *getproxy PARAMS ((struct url *));
+static char *getproxy (struct url *);
/* Retrieve the given URL. Decides which loop to call -- HTTP, FTP,
FTP, proxy, etc. */
uerr_t
retrieve_url (const char *origurl, char **file, char **newloc,
- const char *refurl, int *dt)
+ const char *refurl, int *dt, bool recursive)
{
uerr_t result;
char *url;
- int location_changed, dummy;
+ bool location_changed;
+ int dummy;
char *mynewloc, *proxy;
struct url *u, *proxy_url;
int up_error_code; /* url parse error code */
char *local_file;
int redirection_count = 0;
- int post_data_suspended = 0;
+ bool post_data_suspended = false;
char *saved_post_data = NULL;
char *saved_post_file_name = NULL;
}
else if (u->scheme == SCHEME_FTP)
{
- /* If this is a redirection, we must not allow recursive FTP
- retrieval, so we save recursion to oldrec, and restore it
- later. */
- int oldrec = opt.recursive;
+ /* If this is a redirection, temporarily turn off opt.ftp_glob
+ and opt.recursive, both being undesirable when following
+ redirects. */
+ bool oldrec = recursive, glob = opt.ftp_glob;
if (redirection_count)
- opt.recursive = 0;
- result = ftp_loop (u, dt, proxy_url);
- opt.recursive = oldrec;
+ oldrec = glob = false;
+
+ result = ftp_loop (u, dt, proxy_url, recursive, glob);
+ recursive = oldrec;
/* There is a possibility of having HTTP being redirected to
FTP. In these cases we must decide whether the text is HTML
newloc_parsed = url_parse (mynewloc, &up_error_code);
if (!newloc_parsed)
{
- logprintf (LOG_NOTQUIET, "%s: %s.\n", mynewloc,
+ logprintf (LOG_NOTQUIET, "%s: %s.\n", escnonprint_uri (mynewloc),
url_error (up_error_code));
url_free (u);
xfree (url);
return result;
}
-/* Find the URLs in the file and call retrieve_url() for each of
- them. If HTML is non-zero, treat the file as HTML, and construct
- the URLs accordingly.
+/* Find the URLs in the file and call retrieve_url() for each of them.
+ If HTML is true, treat the file as HTML, and construct the URLs
+ accordingly.
If opt.recursive is set, call retrieve_tree() for each file. */
uerr_t
-retrieve_from_file (const char *file, int html, int *count)
+retrieve_from_file (const char *file, bool html, int *count)
{
uerr_t status;
struct urlpos *url_list, *cur_url;
break;
}
if ((opt.recursive || opt.page_requisites)
- && cur_url->url->scheme != SCHEME_FTP)
- status = retrieve_tree (cur_url->url->url);
+ && (cur_url->url->scheme != SCHEME_FTP || getproxy (cur_url->url)))
+ {
+ int old_follow_ftp = opt.follow_ftp;
+
+ /* Turn opt.follow_ftp on in case of recursive FTP retrieval */
+ if (cur_url->url->scheme == SCHEME_FTP)
+ opt.follow_ftp = 1;
+
+ status = retrieve_tree (cur_url->url->url);
+
+ opt.follow_ftp = old_follow_ftp;
+ }
else
- status = retrieve_url (cur_url->url->url, &filename, &new_file, NULL, &dt);
+ status = retrieve_url (cur_url->url->url, &filename, &new_file, NULL, &dt, opt.recursive);
if (filename && opt.delete_after && file_exists_p (filename))
{
- DEBUGP (("Removing file due to --delete-after in"
- " retrieve_from_file():\n"));
+ DEBUGP (("\
+Removing file due to --delete-after in retrieve_from_file():\n"));
logprintf (LOG_VERBOSE, _("Removing %s.\n"), filename);
if (unlink (filename))
logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
void
sleep_between_retrievals (int count)
{
- static int first_retrieval = 1;
+ static bool first_retrieval = true;
if (first_retrieval)
{
/* Don't sleep before the very first retrieval. */
- first_retrieval = 0;
+ first_retrieval = false;
return;
}
else
{
/* Sleep a random amount of time averaging in opt.wait
- seconds. The sleeping amount ranges from 0 to
- opt.wait*2, inclusive. */
- double waitsecs = 2 * opt.wait * random_float ();
+ seconds. The sleeping amount ranges from 0.5*opt.wait to
+ 1.5*opt.wait. */
+ double waitsecs = (0.5 + random_float ()) * opt.wait;
DEBUGP (("sleep_between_retrievals: avg=%f,sleep=%f\n",
opt.wait, waitsecs));
xsleep (waitsecs);
int maxlen = strlen (fname) + 1 + numdigit (opt.backups) + 1;
char *from = (char *)alloca (maxlen);
char *to = (char *)alloca (maxlen);
- struct stat sb;
+ struct_stat sb;
int i;
if (stat (fname, &sb) == 0)
rename(fname, to);
}
-static int no_proxy_match PARAMS ((const char *, const char **));
+static bool no_proxy_match (const char *, const char **);
/* Return the URL of the proxy appropriate for url U. */
if (!opt.use_proxy)
return NULL;
- if (!no_proxy_match (u->host, (const char **)opt.no_proxy))
+ if (no_proxy_match (u->host, (const char **)opt.no_proxy))
return NULL;
switch (u->scheme)
return proxy;
}
+/* Returns true if URL would be downloaded through a proxy. */
+
+bool
+url_uses_proxy (const char *url)
+{
+ bool ret;
+ struct url *u = url_parse (url, NULL);
+ if (!u)
+ return false;
+ ret = getproxy (u) != NULL;
+ url_free (u);
+ return ret;
+}
+
/* Should a host be accessed through proxy, concerning no_proxy? */
-int
+static bool
no_proxy_match (const char *host, const char **no_proxy)
{
if (!no_proxy)
- return 1;
+ return false;
else
- return !sufmatch (no_proxy, host);
+ return sufmatch (no_proxy, host);
}
projects / wget / blobdiff