#include <stdio.h>
#include <stdlib.h>
-#include <sys/types.h>
#ifdef HAVE_UNISTD_H
# include <unistd.h>
#endif /* HAVE_UNISTD_H */
#include <errno.h>
-#ifdef HAVE_STRING_H
-# include <string.h>
-#else
-# include <strings.h>
-#endif /* HAVE_STRING_H */
+#include <string.h>
#include <assert.h>
#include "wget.h"
#include "url.h"
#include "recur.h"
#include "ftp.h"
+#include "http.h"
#include "host.h"
#include "connect.h"
#include "hash.h"
#include "convert.h"
+#include "ptimer.h"
-#ifdef HAVE_SSL
-# include "gen_sslfunc.h" /* for ssl_iread */
-#endif
-
-#ifndef errno
-extern int errno;
-#endif
+/* Total size of downloaded files. Used to enforce quota. */
+SUM_SIZE_INT total_downloaded_bytes;
-/* See the comment in gethttp() why this is needed. */
-int global_download_count;
+/* Total download time in milliseconds. */
+double total_download_time;
-/* Total size of downloaded files. Used to enforce quota. */
-LARGE_INT total_downloaded_bytes;
+/* If non-NULL, the stream to which output should be written. This
+ stream is initialized when `-O' is used. */
+FILE *output_stream;
+/* Whether output_document is a regular file we can manipulate,
+ i.e. not `-' or a device file. */
+bool output_stream_regular;
\f
static struct {
- long chunk_bytes;
+ wgint chunk_bytes;
double chunk_start;
double sleep_adjust;
} limit_data;
{
limit_data.chunk_bytes = 0;
limit_data.chunk_start = 0;
+ limit_data.sleep_adjust = 0;
}
/* Limit the bandwidth by pausing the download for an amount of time.
is the timer that started at the beginning of download. */
static void
-limit_bandwidth (long bytes, struct wget_timer *timer)
+limit_bandwidth (wgint bytes, struct ptimer *timer)
{
- double delta_t = wtimer_read (timer) - limit_data.chunk_start;
+ double delta_t = ptimer_read (timer) - limit_data.chunk_start;
double expected;
limit_data.chunk_bytes += bytes;
double t0, t1;
if (slp < 200)
{
- DEBUGP (("deferring a %.2f ms sleep (%ld/%.2f).\n",
- slp, limit_data.chunk_bytes, delta_t));
+ DEBUGP (("deferring a %.2f ms sleep (%s/%.2f).\n",
+ slp, number_to_static_string (limit_data.chunk_bytes),
+ delta_t));
return;
}
- DEBUGP (("\nsleeping %.2f ms for %ld bytes, adjust %.2f ms\n",
- slp, limit_data.chunk_bytes, limit_data.sleep_adjust));
+ DEBUGP (("\nsleeping %.2f ms for %s bytes, adjust %.2f ms\n",
+ slp, number_to_static_string (limit_data.chunk_bytes),
+ limit_data.sleep_adjust));
- t0 = wtimer_read (timer);
+ t0 = ptimer_read (timer);
xsleep (slp / 1000);
- wtimer_update (timer);
- t1 = wtimer_read (timer);
+ t1 = ptimer_measure (timer);
/* Due to scheduling, we probably slept slightly longer (or
shorter) than desired. Calculate the difference between the
desired and the actual sleep, and adjust the next sleep by
that amount. */
limit_data.sleep_adjust = slp - (t1 - t0);
+ /* If sleep_adjust is very large, it's likely due to suspension
+ and not clock inaccuracy. Don't enforce those. */
+ if (limit_data.sleep_adjust > 500)
+ limit_data.sleep_adjust = 500;
+ else if (limit_data.sleep_adjust < -500)
+ limit_data.sleep_adjust = -500;
}
limit_data.chunk_bytes = 0;
- limit_data.chunk_start = wtimer_read (timer);
+ limit_data.chunk_start = ptimer_read (timer);
}
#ifndef MIN
# define MIN(i, j) ((i) <= (j) ? (i) : (j))
#endif
+/* Write data in BUF to OUT. However, if *SKIP is non-zero, skip that
+ amount of data and decrease SKIP. Increment *TOTAL by the amount
+ of data written. */
+
+static int
+write_data (FILE *out, const char *buf, int bufsize, wgint *skip,
+ wgint *written)
+{
+ if (!out)
+ return 1;
+ if (*skip > bufsize)
+ {
+ *skip -= bufsize;
+ return 1;
+ }
+ if (*skip)
+ {
+ buf += *skip;
+ bufsize -= *skip;
+ *skip = 0;
+ if (bufsize == 0)
+ return 1;
+ }
+
+ fwrite (buf, 1, bufsize, out);
+ *written += bufsize;
+
+ /* Immediately flush the downloaded data. This should not hinder
+ performance: fast downloads will arrive in large 16K chunks
+ (which stdio would write out immediately anyway), and slow
+ downloads wouldn't be limited by disk speed. */
+ fflush (out);
+ return !ferror (out);
+}
+
/* Read the contents of file descriptor FD until it the connection
terminates or a read error occurs. The data is read in portions of
up to 16K and written to OUT as it arrives. If opt.verbose is set,
the progress is shown.
TOREAD is the amount of data expected to arrive, normally only used
- by the progress gauge. However, if EXACT is set, no more than
- TOREAD octets will be read.
+ by the progress gauge.
STARTPOS is the position from which the download starts, used by
- the progress gauge. The amount of data read gets stored to
- *AMOUNT_READ. The time it took to download the data (in
- milliseconds) is stored to *ELAPSED.
+ the progress gauge. If QTYREAD is non-NULL, the value it points to
+ is incremented by the amount of data read from the network. If
+ QTYWRITTEN is non-NULL, the value it points to is incremented by
+ the amount of data written to disk. The time it took to download
+ the data (in milliseconds) is stored to ELAPSED.
The function exits and returns the amount of data read. In case of
error while reading data, -1 is returned. In case of error while
writing data, -2 is returned. */
int
-fd_read_body (int fd, FILE *out, long toread, int exact, long startpos,
- long *amount_read, double *elapsed)
+fd_read_body (int fd, FILE *out, wgint toread, wgint startpos,
+ wgint *qtyread, wgint *qtywritten, double *elapsed, int flags)
{
int ret = 0;
static char dlbuf[16384];
int dlbufsize = sizeof (dlbuf);
- struct wget_timer *timer = NULL;
+ struct ptimer *timer = NULL;
double last_successful_read_tm = 0;
/* The progress gauge, set according to the user preferences. */
continually update the display. When true, smaller timeout
values are used so that the gauge can update the display when
data arrives slowly. */
- int progress_interactive = 0;
+ bool progress_interactive = false;
+
+ bool exact = !!(flags & rb_read_exactly);
+ wgint skip = 0;
- *amount_read = 0;
+ /* How much data we've read/written. */
+ wgint sum_read = 0;
+ wgint sum_written = 0;
+
+ if (flags & rb_skip_startpos)
+ skip = startpos;
if (opt.verbose)
{
- progress = progress_create (startpos, toread);
+ /* If we're skipping STARTPOS bytes, pass 0 as the INITIAL
+ argument to progress_create because the indicator doesn't
+ (yet) know about "skipping" data. */
+ progress = progress_create (skip ? 0 : startpos, startpos + toread);
progress_interactive = progress_interactive_p (progress);
}
the timer. */
if (progress || opt.limit_rate || elapsed)
{
- timer = wtimer_new ();
+ timer = ptimer_new ();
last_successful_read_tm = 0;
}
means that it is unknown how much data is to arrive. However, if
EXACT is set, then toread==0 means what it says: that no data
should be read. */
- while (!exact || (*amount_read < toread))
+ while (!exact || (sum_read < toread))
{
- int rdsize = exact ? MIN (toread - *amount_read, dlbufsize) : dlbufsize;
+ int rdsize = exact ? MIN (toread - sum_read, dlbufsize) : dlbufsize;
double tmout = opt.read_timeout;
if (progress_interactive)
{
- double waittm;
/* For interactive progress gauges, always specify a ~1s
timeout, so that the gauge can be updated regularly even
when the data arrives very slowly or stalls. */
tmout = 0.95;
- waittm = (wtimer_read (timer) - last_successful_read_tm) / 1000;
- if (waittm + tmout > opt.read_timeout)
+ if (opt.read_timeout)
{
- /* Don't allow waiting time to exceed read timeout. */
- tmout = opt.read_timeout - waittm;
- if (tmout < 0)
+ double waittm;
+ waittm = (ptimer_read (timer) - last_successful_read_tm) / 1000;
+ if (waittm + tmout > opt.read_timeout)
{
- /* We've already exceeded the timeout. */
- ret = -1, errno = ETIMEDOUT;
- break;
+ /* Don't let total idle time exceed read timeout. */
+ tmout = opt.read_timeout - waittm;
+ if (tmout < 0)
+ {
+ /* We've already exceeded the timeout. */
+ ret = -1, errno = ETIMEDOUT;
+ break;
+ }
}
}
}
ret = fd_read (fd, dlbuf, rdsize, tmout);
- if (ret == 0 || (ret < 0 && errno != ETIMEDOUT))
- break;
- else if (ret < 0)
- ret = 0; /* timeout */
+ if (progress_interactive && ret < 0 && errno == ETIMEDOUT)
+ ret = 0; /* interactive timeout, handled above */
+ else if (ret <= 0)
+ break; /* EOF or read error */
if (progress || opt.limit_rate)
{
- wtimer_update (timer);
+ ptimer_measure (timer);
if (ret > 0)
- last_successful_read_tm = wtimer_read (timer);
+ last_successful_read_tm = ptimer_read (timer);
}
- if (ret > 0 && out != NULL)
+ if (ret > 0)
{
- fwrite (dlbuf, 1, ret, out);
- /* Immediately flush the downloaded data. This should not
- hinder performance: fast downloads will arrive in large
- 16K chunks (which stdio would write out anyway), and slow
- downloads wouldn't be limited by disk speed. */
- fflush (out);
- if (ferror (out))
+ sum_read += ret;
+ if (!write_data (out, dlbuf, ret, &skip, &sum_written))
{
ret = -2;
goto out;
if (opt.limit_rate)
limit_bandwidth (ret, timer);
- *amount_read += ret;
if (progress)
- progress_update (progress, ret, wtimer_read (timer));
+ progress_update (progress, ret, ptimer_read (timer));
#ifdef WINDOWS
- if (toread > 0)
+ if (toread > 0 && !opt.quiet)
ws_percenttitle (100.0 *
- (startpos + *amount_read) / (startpos + toread));
+ (startpos + sum_read) / (startpos + toread));
#endif
}
if (ret < -1)
out:
if (progress)
- progress_finish (progress, wtimer_read (timer));
+ progress_finish (progress, ptimer_read (timer));
+
if (elapsed)
- *elapsed = wtimer_read (timer);
+ *elapsed = ptimer_read (timer);
if (timer)
- wtimer_delete (timer);
+ ptimer_destroy (timer);
+
+ if (qtyread)
+ *qtyread += sum_read;
+ if (qtywritten)
+ *qtywritten += sum_written;
return ret;
}
a read. If the read returns a different amount of data, the
process is retried until all data arrives safely.
- BUFSIZE is the size of the initial buffer expected to read all the
- data in the typical case.
+ SIZEHINT is the buffer size sufficient to hold all the data in the
+ typical case (it is used as the initial buffer size). MAXSIZE is
+ the maximum amount of memory this function is allowed to allocate,
+ or 0 if no upper limit is to be enforced.
This function should be used as a building block for other
functions -- see fd_read_line as a simple example. */
char *
-fd_read_hunk (int fd, hunk_terminator_t hunk_terminator, int bufsize)
+fd_read_hunk (int fd, hunk_terminator_t terminator, long sizehint, long maxsize)
{
+ long bufsize = sizehint;
char *hunk = xmalloc (bufsize);
int tail = 0; /* tail position in HUNK */
+ assert (maxsize >= bufsize);
+
while (1)
{
const char *end;
xfree (hunk);
return NULL;
}
- end = hunk_terminator (hunk, tail, pklen);
+ end = terminator (hunk, tail, pklen);
if (end)
{
/* The data contains the terminator: we'll drain the data up
if (tail == bufsize - 1)
{
+ /* Double the buffer size, but refuse to allocate more than
+ MAXSIZE bytes. */
+ if (maxsize && bufsize >= maxsize)
+ {
+ xfree (hunk);
+ errno = ENOMEM;
+ return NULL;
+ }
bufsize <<= 1;
+ if (maxsize && bufsize > maxsize)
+ bufsize = maxsize;
hunk = xrealloc (hunk, bufsize);
}
}
return NULL;
}
+/* The maximum size of the single line we agree to accept. This is
+ not meant to impose an arbitrary limit, but to protect the user
+ from Wget slurping up available memory upon encountering malicious
+ or buggy server output. Define it to 0 to remove the limit. */
+#define FD_READ_LINE_MAX 4096
+
/* Read one line from FD and return it. The line is allocated using
- malloc.
+ malloc, but is never larger than FD_READ_LINE_MAX.
If an error occurs, or if no data can be read, NULL is returned.
In the former case errno indicates the error condition, and in the
char *
fd_read_line (int fd)
{
- return fd_read_hunk (fd, line_terminator, 128);
+ return fd_read_hunk (fd, line_terminator, 128, FD_READ_LINE_MAX);
}
\f
-/* Return a printed representation of the download rate, as
- appropriate for the speed. If PAD is non-zero, strings will be
- padded to the width of 7 characters (xxxx.xx). */
-char *
-retr_rate (long bytes, double msecs, int pad)
+/* Return a printed representation of the download rate, along with
+ the units appropriate for the download speed. */
+
+const char *
+retr_rate (wgint bytes, double msecs)
{
static char res[20];
- static char *rate_names[] = {"B/s", "KB/s", "MB/s", "GB/s" };
+ static const char *rate_names[] = {"B/s", "KB/s", "MB/s", "GB/s" };
int units = 0;
double dlrate = calc_rate (bytes, msecs, &units);
- sprintf (res, pad ? "%7.2f %s" : "%.2f %s", dlrate, rate_names[units]);
+ sprintf (res, "%.2f %s", dlrate, rate_names[units]);
return res;
}
UNITS is zero for B/s, one for KB/s, two for MB/s, and three for
GB/s. */
double
-calc_rate (long bytes, double msecs, int *units)
+calc_rate (wgint bytes, double msecs, int *units)
{
double dlrate;
if (msecs == 0)
/* If elapsed time is exactly zero, it means we're under the
- granularity of the timer. This often happens on systems that
- use time() for the timer. */
- msecs = wtimer_granularity ();
+ resolution of the timer. This can easily happen on systems
+ that use time() for the timer. Since the interval lies between
+ 0 and the timer's resolution, assume half the resolution. */
+ msecs = ptimer_resolution () / 2.0;
- dlrate = (double)1000 * bytes / msecs;
+ dlrate = 1000.0 * bytes / msecs;
if (dlrate < 1024.0)
*units = 0;
else if (dlrate < 1024.0 * 1024.0)
#define MAX_REDIRECTIONS 20
#define SUSPEND_POST_DATA do { \
- post_data_suspended = 1; \
+ post_data_suspended = true; \
saved_post_data = opt.post_data; \
saved_post_file_name = opt.post_file_name; \
opt.post_data = NULL; \
{ \
opt.post_data = saved_post_data; \
opt.post_file_name = saved_post_file_name; \
- post_data_suspended = 0; \
+ post_data_suspended = false; \
} \
} while (0)
-static char *getproxy PARAMS ((struct url *));
+static char *getproxy (struct url *);
/* Retrieve the given URL. Decides which loop to call -- HTTP, FTP,
FTP, proxy, etc. */
{
uerr_t result;
char *url;
- int location_changed, dummy;
+ bool location_changed;
+ int dummy;
char *mynewloc, *proxy;
struct url *u, *proxy_url;
int up_error_code; /* url parse error code */
char *local_file;
int redirection_count = 0;
- int post_data_suspended = 0;
+ bool post_data_suspended = false;
char *saved_post_data = NULL;
char *saved_post_file_name = NULL;
}
else if (u->scheme == SCHEME_FTP)
{
- /* If this is a redirection, we must not allow recursive FTP
- retrieval, so we save recursion to oldrec, and restore it
- later. */
- int oldrec = opt.recursive;
+ /* If this is a redirection, temporarily turn off opt.ftp_glob
+ and opt.recursive, both being undesirable when following
+ redirects. */
+ bool oldrec = opt.recursive, oldglob = opt.ftp_glob;
if (redirection_count)
- opt.recursive = 0;
+ opt.recursive = opt.ftp_glob = false;
+
result = ftp_loop (u, dt, proxy_url);
opt.recursive = oldrec;
+ opt.ftp_glob = oldglob;
/* There is a possibility of having HTTP being redirected to
FTP. In these cases we must decide whether the text is HTML
newloc_parsed = url_parse (mynewloc, &up_error_code);
if (!newloc_parsed)
{
- logprintf (LOG_NOTQUIET, "%s: %s.\n", mynewloc,
+ logprintf (LOG_NOTQUIET, "%s: %s.\n", escnonprint_uri (mynewloc),
url_error (up_error_code));
url_free (u);
xfree (url);
xfree (url);
}
- ++global_download_count;
RESTORE_POST_DATA;
return result;
}
-/* Find the URLs in the file and call retrieve_url() for each of
- them. If HTML is non-zero, treat the file as HTML, and construct
- the URLs accordingly.
+/* Find the URLs in the file and call retrieve_url() for each of them.
+ If HTML is true, treat the file as HTML, and construct the URLs
+ accordingly.
If opt.recursive is set, call retrieve_tree() for each file. */
uerr_t
-retrieve_from_file (const char *file, int html, int *count)
+retrieve_from_file (const char *file, bool html, int *count)
{
uerr_t status;
struct urlpos *url_list, *cur_url;
if (filename && opt.delete_after && file_exists_p (filename))
{
- DEBUGP (("Removing file due to --delete-after in"
- " retrieve_from_file():\n"));
+ DEBUGP (("\
+Removing file due to --delete-after in retrieve_from_file():\n"));
logprintf (LOG_VERBOSE, _("Removing %s.\n"), filename);
if (unlink (filename))
logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
void
sleep_between_retrievals (int count)
{
- static int first_retrieval = 1;
+ static bool first_retrieval = true;
if (first_retrieval)
{
/* Don't sleep before the very first retrieval. */
- first_retrieval = 0;
+ first_retrieval = false;
return;
}
int maxlen = strlen (fname) + 1 + numdigit (opt.backups) + 1;
char *from = (char *)alloca (maxlen);
char *to = (char *)alloca (maxlen);
- struct stat sb;
+ struct_stat sb;
int i;
if (stat (fname, &sb) == 0)
rename(fname, to);
}
-static int no_proxy_match PARAMS ((const char *, const char **));
+static bool no_proxy_match (const char *, const char **);
/* Return the URL of the proxy appropriate for url U. */
}
/* Should a host be accessed through proxy, concerning no_proxy? */
-int
+static bool
no_proxy_match (const char *host, const char **no_proxy)
{
if (!no_proxy)
- return 1;
+ return true;
else
return !sufmatch (no_proxy, host);
}