/* File retrieval.
- Copyright (C) 1995, 1996, 1997, 1998, 2000, 2001 Free Software Foundation, Inc.
+ Copyright (C) 1996-2006 Free Software Foundation, Inc.
This file is part of GNU Wget.
GNU Wget is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or (at
+the Free Software Foundation; either version 3 of the License, or (at
your option) any later version.
GNU Wget is distributed in the hope that it will be useful,
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
-along with Wget; if not, write to the Free Software
-Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+along with Wget. If not, see <http://www.gnu.org/licenses/>.
In addition, as a special exception, the Free Software Foundation
gives permission to link the code of its release of Wget with the
#include <stdio.h>
#include <stdlib.h>
-#include <sys/types.h>
#ifdef HAVE_UNISTD_H
# include <unistd.h>
#endif /* HAVE_UNISTD_H */
#include <errno.h>
-#ifdef HAVE_STRING_H
-# include <string.h>
-#else
-# include <strings.h>
-#endif /* HAVE_STRING_H */
+#include <string.h>
#include <assert.h>
#include "wget.h"
#include "url.h"
#include "recur.h"
#include "ftp.h"
+#include "http.h"
#include "host.h"
#include "connect.h"
#include "hash.h"
#include "convert.h"
#include "ptimer.h"
-#ifndef errno
-extern int errno;
-#endif
-
/* Total size of downloaded files. Used to enforce quota. */
-LARGE_INT total_downloaded_bytes;
+SUM_SIZE_INT total_downloaded_bytes;
+
+/* Total download time in seconds. */
+double total_download_time;
/* If non-NULL, the stream to which output should be written. This
stream is initialized when `-O' is used. */
/* Whether output_document is a regular file we can manipulate,
i.e. not `-' or a device file. */
-int output_stream_regular;
+bool output_stream_regular;
\f
static struct {
wgint chunk_bytes;
static void
limit_bandwidth_reset (void)
{
- limit_data.chunk_bytes = 0;
- limit_data.chunk_start = 0;
+ xzero (limit_data);
}
/* Limit the bandwidth by pausing the download for an amount of time.
/* Calculate the amount of time we expect downloading the chunk
should take. If in reality it took less time, sleep to
compensate for the difference. */
- expected = 1000.0 * limit_data.chunk_bytes / opt.limit_rate;
+ expected = (double) limit_data.chunk_bytes / opt.limit_rate;
if (expected > delta_t)
{
double slp = expected - delta_t + limit_data.sleep_adjust;
double t0, t1;
- if (slp < 200)
+ if (slp < 0.2)
{
DEBUGP (("deferring a %.2f ms sleep (%s/%.2f).\n",
- slp, number_to_static_string (limit_data.chunk_bytes),
+ slp * 1000, number_to_static_string (limit_data.chunk_bytes),
delta_t));
return;
}
DEBUGP (("\nsleeping %.2f ms for %s bytes, adjust %.2f ms\n",
- slp, number_to_static_string (limit_data.chunk_bytes),
+ slp * 1000, number_to_static_string (limit_data.chunk_bytes),
limit_data.sleep_adjust));
t0 = ptimer_read (timer);
- xsleep (slp / 1000);
+ xsleep (slp);
t1 = ptimer_measure (timer);
/* Due to scheduling, we probably slept slightly longer (or
desired and the actual sleep, and adjust the next sleep by
that amount. */
limit_data.sleep_adjust = slp - (t1 - t0);
+ /* If sleep_adjust is very large, it's likely due to suspension
+ and not clock inaccuracy. Don't enforce those. */
+ if (limit_data.sleep_adjust > 0.5)
+ limit_data.sleep_adjust = 0.5;
+ else if (limit_data.sleep_adjust < -0.5)
+ limit_data.sleep_adjust = -0.5;
}
limit_data.chunk_bytes = 0;
is incremented by the amount of data read from the network. If
QTYWRITTEN is non-NULL, the value it points to is incremented by
the amount of data written to disk. The time it took to download
- the data (in milliseconds) is stored to ELAPSED.
+ the data is stored to ELAPSED.
The function exits and returns the amount of data read. In case of
error while reading data, -1 is returned. In case of error while
continually update the display. When true, smaller timeout
values are used so that the gauge can update the display when
data arrives slowly. */
- int progress_interactive = 0;
+ bool progress_interactive = false;
- int exact = flags & rb_read_exactly;
+ bool exact = !!(flags & rb_read_exactly);
wgint skip = 0;
/* How much data we've read/written. */
if (opt.read_timeout)
{
double waittm;
- waittm = (ptimer_read (timer) - last_successful_read_tm) / 1000;
+ waittm = ptimer_read (timer) - last_successful_read_tm;
if (waittm + tmout > opt.read_timeout)
{
/* Don't let total idle time exceed read timeout. */
if (!write_data (out, dlbuf, ret, &skip, &sum_written))
{
ret = -2;
- goto out_;
+ goto out;
}
}
if (ret < -1)
ret = -1;
- out_:
+ out:
if (progress)
progress_finish (progress, ptimer_read (timer));
return ret;
}
\f
-/* Read a hunk of data from FD, up until a terminator. The terminator
- is whatever the TERMINATOR function determines it to be; for
- example, it can be a line of data, or the head of an HTTP response.
- The function returns the data read allocated with malloc.
-
- In case of error, NULL is returned. In case of EOF and no data
- read, NULL is returned and errno set to 0. In case of EOF with
- data having been read, the data is returned, but it will
- (obviously) not contain the terminator.
+/* Read a hunk of data from FD, up until a terminator. The hunk is
+ limited by whatever the TERMINATOR callback chooses as its
+ terminator. For example, if terminator stops at newline, the hunk
+ will consist of a line of data; if terminator stops at two
+ newlines, it can be used to read the head of an HTTP response.
+ Upon determining the boundary, the function returns the data (up to
+ the terminator) in malloc-allocated storage.
+
+ In case of read error, NULL is returned. In case of EOF and no
+ data read, NULL is returned and errno set to 0. In case of having
+ read some data, but encountering EOF before seeing the terminator,
+ the data that has been read is returned, but it will (obviously)
+ not contain the terminator.
+
+ The TERMINATOR function is called with three arguments: the
+ beginning of the data read so far, the beginning of the current
+ block of peeked-at data, and the length of the current block.
+ Depending on its needs, the function is free to choose whether to
+ analyze all data or just the newly arrived data. If TERMINATOR
+ returns NULL, it means that the terminator has not been seen.
+ Otherwise it should return a pointer to the charactre immediately
+ following the terminator.
The idea is to be able to read a line of input, or otherwise a hunk
of text, such as the head of an HTTP request, without crossing the
boundary, so that the next call to fd_read etc. reads the data
after the hunk. To achieve that, this function does the following:
- 1. Peek at available data.
+ 1. Peek at incoming data.
2. Determine whether the peeked data, along with the previously
read data, includes the terminator.
/* First, peek at the available data. */
- pklen = fd_peek (fd, hunk + tail, bufsize - 1 - tail, -1.0);
+ pklen = fd_peek (fd, hunk + tail, bufsize - 1 - tail, -1);
if (pklen < 0)
{
xfree (hunk);
return NULL;
}
- end = terminator (hunk, tail, pklen);
+ end = terminator (hunk, hunk + tail, pklen);
if (end)
{
/* The data contains the terminator: we'll drain the data up
to the end of the terminator. */
remain = end - (hunk + tail);
+ assert (remain >= 0);
if (remain == 0)
{
/* No more data needs to be read. */
how much data we'll get. (Some TCP stacks are notorious for
read returning less data than the previous MSG_PEEK.) */
- rdlen = fd_read (fd, hunk + tail, remain, 0.0);
+ rdlen = fd_read (fd, hunk + tail, remain, 0);
if (rdlen < 0)
{
xfree_null (hunk);
}
static const char *
-line_terminator (const char *hunk, int oldlen, int peeklen)
+line_terminator (const char *start, const char *peeked, int peeklen)
{
- const char *p = memchr (hunk + oldlen, '\n', peeklen);
+ const char *p = memchr (peeked, '\n', peeklen);
if (p)
- /* p+1 because we want the line to include '\n' */
+ /* p+1 because the line must include '\n' */
return p + 1;
return NULL;
}
return fd_read_hunk (fd, line_terminator, 128, FD_READ_LINE_MAX);
}
\f
-/* Return a printed representation of the download rate, as
- appropriate for the speed. If PAD is non-zero, strings will be
- padded to the width of 7 characters (xxxx.xx). */
-char *
-retr_rate (wgint bytes, double msecs, int pad)
+/* Return a printed representation of the download rate, along with
+ the units appropriate for the download speed. */
+
+const char *
+retr_rate (wgint bytes, double secs)
{
static char res[20];
static const char *rate_names[] = {"B/s", "KB/s", "MB/s", "GB/s" };
- int units = 0;
+ int units;
- double dlrate = calc_rate (bytes, msecs, &units);
- sprintf (res, pad ? "%7.2f %s" : "%.2f %s", dlrate, rate_names[units]);
+ double dlrate = calc_rate (bytes, secs, &units);
+ /* Use more digits for smaller numbers (regardless of unit used),
+ e.g. "1022", "247", "12.5", "2.38". */
+ sprintf (res, "%.*f %s",
+ dlrate >= 99.95 ? 0 : dlrate >= 9.995 ? 1 : 2,
+ dlrate, rate_names[units]);
return res;
}
UNITS is zero for B/s, one for KB/s, two for MB/s, and three for
GB/s. */
+
double
-calc_rate (wgint bytes, double msecs, int *units)
+calc_rate (wgint bytes, double secs, int *units)
{
double dlrate;
- assert (msecs >= 0);
+ assert (secs >= 0);
assert (bytes >= 0);
- if (msecs == 0)
+ if (secs == 0)
/* If elapsed time is exactly zero, it means we're under the
resolution of the timer. This can easily happen on systems
that use time() for the timer. Since the interval lies between
0 and the timer's resolution, assume half the resolution. */
- msecs = ptimer_resolution () / 2.0;
+ secs = ptimer_resolution () / 2.0;
- dlrate = 1000.0 * bytes / msecs;
+ dlrate = bytes / secs;
if (dlrate < 1024.0)
*units = 0;
else if (dlrate < 1024.0 * 1024.0)
#define MAX_REDIRECTIONS 20
#define SUSPEND_POST_DATA do { \
- post_data_suspended = 1; \
+ post_data_suspended = true; \
saved_post_data = opt.post_data; \
saved_post_file_name = opt.post_file_name; \
opt.post_data = NULL; \
{ \
opt.post_data = saved_post_data; \
opt.post_file_name = saved_post_file_name; \
- post_data_suspended = 0; \
+ post_data_suspended = false; \
} \
} while (0)
-static char *getproxy PARAMS ((struct url *));
+static char *getproxy (struct url *);
/* Retrieve the given URL. Decides which loop to call -- HTTP, FTP,
FTP, proxy, etc. */
uerr_t
retrieve_url (const char *origurl, char **file, char **newloc,
- const char *refurl, int *dt)
+ const char *refurl, int *dt, bool recursive)
{
uerr_t result;
char *url;
- int location_changed, dummy;
+ bool location_changed;
+ int dummy;
char *mynewloc, *proxy;
struct url *u, *proxy_url;
int up_error_code; /* url parse error code */
char *local_file;
int redirection_count = 0;
- int post_data_suspended = 0;
+ bool post_data_suspended = false;
char *saved_post_data = NULL;
char *saved_post_file_name = NULL;
}
else if (u->scheme == SCHEME_FTP)
{
- /* If this is a redirection, we must not allow recursive FTP
- retrieval, so we save recursion to oldrec, and restore it
- later. */
- int oldrec = opt.recursive;
+ /* If this is a redirection, temporarily turn off opt.ftp_glob
+ and opt.recursive, both being undesirable when following
+ redirects. */
+ bool oldrec = recursive, glob = opt.ftp_glob;
if (redirection_count)
- opt.recursive = 0;
- result = ftp_loop (u, dt, proxy_url);
- opt.recursive = oldrec;
+ oldrec = glob = false;
+
+ result = ftp_loop (u, dt, proxy_url, recursive, glob);
+ recursive = oldrec;
/* There is a possibility of having HTTP being redirected to
FTP. In these cases we must decide whether the text is HTML
return result;
}
-/* Find the URLs in the file and call retrieve_url() for each of
- them. If HTML is non-zero, treat the file as HTML, and construct
- the URLs accordingly.
+/* Find the URLs in the file and call retrieve_url() for each of them.
+ If HTML is true, treat the file as HTML, and construct the URLs
+ accordingly.
If opt.recursive is set, call retrieve_tree() for each file. */
uerr_t
-retrieve_from_file (const char *file, int html, int *count)
+retrieve_from_file (const char *file, bool html, int *count)
{
uerr_t status;
struct urlpos *url_list, *cur_url;
break;
}
if ((opt.recursive || opt.page_requisites)
- && cur_url->url->scheme != SCHEME_FTP)
- status = retrieve_tree (cur_url->url->url);
+ && (cur_url->url->scheme != SCHEME_FTP || getproxy (cur_url->url)))
+ {
+ int old_follow_ftp = opt.follow_ftp;
+
+ /* Turn opt.follow_ftp on in case of recursive FTP retrieval */
+ if (cur_url->url->scheme == SCHEME_FTP)
+ opt.follow_ftp = 1;
+
+ status = retrieve_tree (cur_url->url->url);
+
+ opt.follow_ftp = old_follow_ftp;
+ }
else
- status = retrieve_url (cur_url->url->url, &filename, &new_file, NULL, &dt);
+ status = retrieve_url (cur_url->url->url, &filename, &new_file, NULL, &dt, opt.recursive);
if (filename && opt.delete_after && file_exists_p (filename))
{
void
sleep_between_retrievals (int count)
{
- static int first_retrieval = 1;
+ static bool first_retrieval = true;
if (first_retrieval)
{
/* Don't sleep before the very first retrieval. */
- first_retrieval = 0;
+ first_retrieval = false;
return;
}
/* If opt.waitretry is specified and this is a retry, wait for
COUNT-1 number of seconds, or for opt.waitretry seconds. */
if (count <= opt.waitretry)
- xsleep (count - 1.0);
+ xsleep (count - 1);
else
xsleep (opt.waitretry);
}
else
{
/* Sleep a random amount of time averaging in opt.wait
- seconds. The sleeping amount ranges from 0 to
- opt.wait*2, inclusive. */
- double waitsecs = 2 * opt.wait * random_float ();
+ seconds. The sleeping amount ranges from 0.5*opt.wait to
+ 1.5*opt.wait. */
+ double waitsecs = (0.5 + random_float ()) * opt.wait;
DEBUGP (("sleep_between_retrievals: avg=%f,sleep=%f\n",
opt.wait, waitsecs));
xsleep (waitsecs);
rename(fname, to);
}
-static int no_proxy_match PARAMS ((const char *, const char **));
+static bool no_proxy_match (const char *, const char **);
/* Return the URL of the proxy appropriate for url U. */
if (!opt.use_proxy)
return NULL;
- if (!no_proxy_match (u->host, (const char **)opt.no_proxy))
+ if (no_proxy_match (u->host, (const char **)opt.no_proxy))
return NULL;
switch (u->scheme)
return proxy;
}
+/* Returns true if URL would be downloaded through a proxy. */
+
+bool
+url_uses_proxy (const char *url)
+{
+ bool ret;
+ struct url *u = url_parse (url, NULL);
+ if (!u)
+ return false;
+ ret = getproxy (u) != NULL;
+ url_free (u);
+ return ret;
+}
+
/* Should a host be accessed through proxy, concerning no_proxy? */
-static int
+static bool
no_proxy_match (const char *host, const char **no_proxy)
{
if (!no_proxy)
- return 1;
+ return false;
else
- return !sufmatch (no_proxy, host);
+ return sufmatch (no_proxy, host);
}
projects / wget / blobdiff