#include <stdio.h>
#include <stdlib.h>
-#include <sys/types.h>
#ifdef HAVE_UNISTD_H
# include <unistd.h>
#endif /* HAVE_UNISTD_H */
#include <errno.h>
-#ifdef HAVE_STRING_H
-# include <string.h>
-#else
-# include <strings.h>
-#endif /* HAVE_STRING_H */
+#include <string.h>
#include <assert.h>
#include "wget.h"
#include "url.h"
#include "recur.h"
#include "ftp.h"
+#include "http.h"
#include "host.h"
#include "connect.h"
#include "hash.h"
#include "convert.h"
-
-#ifdef HAVE_SSL
-# include "gen_sslfunc.h" /* for ssl_iread */
-#endif
-
-#ifndef errno
-extern int errno;
-#endif
+#include "ptimer.h"
/* Total size of downloaded files. Used to enforce quota. */
-LARGE_INT total_downloaded_bytes;
+SUM_SIZE_INT total_downloaded_bytes;
+
+/* Total download time in milliseconds. */
+double total_download_time;
/* If non-NULL, the stream to which output should be written. This
stream is initialized when `-O' is used. */
/* Whether output_document is a regular file we can manipulate,
i.e. not `-' or a device file. */
-int output_stream_regular;
+bool output_stream_regular;
\f
static struct {
wgint chunk_bytes;
{
limit_data.chunk_bytes = 0;
limit_data.chunk_start = 0;
+ limit_data.sleep_adjust = 0;
}
/* Limit the bandwidth by pausing the download for an amount of time.
is the timer that started at the beginning of download. */
static void
-limit_bandwidth (wgint bytes, struct wget_timer *timer)
+limit_bandwidth (wgint bytes, struct ptimer *timer)
{
- double delta_t = wtimer_read (timer) - limit_data.chunk_start;
+ double delta_t = ptimer_read (timer) - limit_data.chunk_start;
double expected;
limit_data.chunk_bytes += bytes;
slp, number_to_static_string (limit_data.chunk_bytes),
limit_data.sleep_adjust));
- t0 = wtimer_read (timer);
+ t0 = ptimer_read (timer);
xsleep (slp / 1000);
- wtimer_update (timer);
- t1 = wtimer_read (timer);
+ t1 = ptimer_measure (timer);
/* Due to scheduling, we probably slept slightly longer (or
shorter) than desired. Calculate the difference between the
desired and the actual sleep, and adjust the next sleep by
that amount. */
limit_data.sleep_adjust = slp - (t1 - t0);
+ /* If sleep_adjust is very large, it's likely due to suspension
+ and not clock inaccuracy. Don't enforce those. */
+ if (limit_data.sleep_adjust > 500)
+ limit_data.sleep_adjust = 500;
+ else if (limit_data.sleep_adjust < -500)
+ limit_data.sleep_adjust = -500;
}
limit_data.chunk_bytes = 0;
- limit_data.chunk_start = wtimer_read (timer);
+ limit_data.chunk_start = ptimer_read (timer);
}
#ifndef MIN
static char dlbuf[16384];
int dlbufsize = sizeof (dlbuf);
- struct wget_timer *timer = NULL;
+ struct ptimer *timer = NULL;
double last_successful_read_tm = 0;
/* The progress gauge, set according to the user preferences. */
continually update the display. When true, smaller timeout
values are used so that the gauge can update the display when
data arrives slowly. */
- int progress_interactive = 0;
+ bool progress_interactive = false;
- int exact = flags & rb_read_exactly;
+ bool exact = !!(flags & rb_read_exactly);
wgint skip = 0;
/* How much data we've read/written. */
the timer. */
if (progress || opt.limit_rate || elapsed)
{
- timer = wtimer_new ();
+ timer = ptimer_new ();
last_successful_read_tm = 0;
}
if (opt.read_timeout)
{
double waittm;
- waittm = (wtimer_read (timer) - last_successful_read_tm) / 1000;
+ waittm = (ptimer_read (timer) - last_successful_read_tm) / 1000;
if (waittm + tmout > opt.read_timeout)
{
/* Don't let total idle time exceed read timeout. */
}
ret = fd_read (fd, dlbuf, rdsize, tmout);
- /* when retrieving from http-proxy wget sometimes does not trust the
- * file length reported by server.
- * this check is to tell wget not to stubbornly try to read again and
- * again until another errno code was received. */
- if ( ret == -1 && errno == ETIMEDOUT && sum_read == toread && toread > 0 )
- break;
-
- if (ret == 0 || (ret < 0 && errno != ETIMEDOUT))
- break; /* read error */
- else if (ret < 0)
- ret = 0; /* read timeout */
+ if (progress_interactive && ret < 0 && errno == ETIMEDOUT)
+ ret = 0; /* interactive timeout, handled above */
+ else if (ret <= 0)
+ break; /* EOF or read error */
if (progress || opt.limit_rate)
{
- wtimer_update (timer);
+ ptimer_measure (timer);
if (ret > 0)
- last_successful_read_tm = wtimer_read (timer);
+ last_successful_read_tm = ptimer_read (timer);
}
if (ret > 0)
limit_bandwidth (ret, timer);
if (progress)
- progress_update (progress, ret, wtimer_read (timer));
+ progress_update (progress, ret, ptimer_read (timer));
#ifdef WINDOWS
if (toread > 0 && !opt.quiet)
ws_percenttitle (100.0 *
out:
if (progress)
- progress_finish (progress, wtimer_read (timer));
+ progress_finish (progress, ptimer_read (timer));
if (elapsed)
- *elapsed = wtimer_read (timer);
+ *elapsed = ptimer_read (timer);
if (timer)
- wtimer_delete (timer);
+ ptimer_destroy (timer);
if (qtyread)
*qtyread += sum_read;
a read. If the read returns a different amount of data, the
process is retried until all data arrives safely.
- BUFSIZE is the size of the initial buffer expected to read all the
- data in the typical case.
+ SIZEHINT is the buffer size sufficient to hold all the data in the
+ typical case (it is used as the initial buffer size). MAXSIZE is
+ the maximum amount of memory this function is allowed to allocate,
+ or 0 if no upper limit is to be enforced.
This function should be used as a building block for other
functions -- see fd_read_line as a simple example. */
char *
-fd_read_hunk (int fd, hunk_terminator_t hunk_terminator, int bufsize)
+fd_read_hunk (int fd, hunk_terminator_t terminator, long sizehint, long maxsize)
{
+ long bufsize = sizehint;
char *hunk = xmalloc (bufsize);
int tail = 0; /* tail position in HUNK */
+ assert (maxsize >= bufsize);
+
while (1)
{
const char *end;
xfree (hunk);
return NULL;
}
- end = hunk_terminator (hunk, tail, pklen);
+ end = terminator (hunk, tail, pklen);
if (end)
{
/* The data contains the terminator: we'll drain the data up
if (tail == bufsize - 1)
{
+ /* Double the buffer size, but refuse to allocate more than
+ MAXSIZE bytes. */
+ if (maxsize && bufsize >= maxsize)
+ {
+ xfree (hunk);
+ errno = ENOMEM;
+ return NULL;
+ }
bufsize <<= 1;
+ if (maxsize && bufsize > maxsize)
+ bufsize = maxsize;
hunk = xrealloc (hunk, bufsize);
}
}
return NULL;
}
+/* The maximum size of the single line we agree to accept. This is
+ not meant to impose an arbitrary limit, but to protect the user
+ from Wget slurping up available memory upon encountering malicious
+ or buggy server output. Define it to 0 to remove the limit. */
+#define FD_READ_LINE_MAX 4096
+
/* Read one line from FD and return it. The line is allocated using
- malloc.
+ malloc, but is never larger than FD_READ_LINE_MAX.
If an error occurs, or if no data can be read, NULL is returned.
In the former case errno indicates the error condition, and in the
char *
fd_read_line (int fd)
{
- return fd_read_hunk (fd, line_terminator, 128);
+ return fd_read_hunk (fd, line_terminator, 128, FD_READ_LINE_MAX);
}
\f
-/* Return a printed representation of the download rate, as
- appropriate for the speed. If PAD is non-zero, strings will be
- padded to the width of 7 characters (xxxx.xx). */
-char *
-retr_rate (wgint bytes, double msecs, int pad)
+/* Return a printed representation of the download rate, along with
+ the units appropriate for the download speed. */
+
+const char *
+retr_rate (wgint bytes, double msecs)
{
static char res[20];
static const char *rate_names[] = {"B/s", "KB/s", "MB/s", "GB/s" };
int units = 0;
double dlrate = calc_rate (bytes, msecs, &units);
- sprintf (res, pad ? "%7.2f %s" : "%.2f %s", dlrate, rate_names[units]);
+ sprintf (res, "%.2f %s", dlrate, rate_names[units]);
return res;
}
if (msecs == 0)
/* If elapsed time is exactly zero, it means we're under the
- granularity of the timer. This often happens on systems that
- use time() for the timer. */
- msecs = wtimer_granularity ();
+ resolution of the timer. This can easily happen on systems
+ that use time() for the timer. Since the interval lies between
+ 0 and the timer's resolution, assume half the resolution. */
+ msecs = ptimer_resolution () / 2.0;
- dlrate = (double)1000 * bytes / msecs;
+ dlrate = 1000.0 * bytes / msecs;
if (dlrate < 1024.0)
*units = 0;
else if (dlrate < 1024.0 * 1024.0)
#define MAX_REDIRECTIONS 20
#define SUSPEND_POST_DATA do { \
- post_data_suspended = 1; \
+ post_data_suspended = true; \
saved_post_data = opt.post_data; \
saved_post_file_name = opt.post_file_name; \
opt.post_data = NULL; \
{ \
opt.post_data = saved_post_data; \
opt.post_file_name = saved_post_file_name; \
- post_data_suspended = 0; \
+ post_data_suspended = false; \
} \
} while (0)
-static char *getproxy PARAMS ((struct url *));
+static char *getproxy (struct url *);
/* Retrieve the given URL. Decides which loop to call -- HTTP, FTP,
FTP, proxy, etc. */
{
uerr_t result;
char *url;
- int location_changed, dummy;
+ bool location_changed;
+ int dummy;
char *mynewloc, *proxy;
struct url *u, *proxy_url;
int up_error_code; /* url parse error code */
char *local_file;
int redirection_count = 0;
- int post_data_suspended = 0;
+ bool post_data_suspended = false;
char *saved_post_data = NULL;
char *saved_post_file_name = NULL;
}
else if (u->scheme == SCHEME_FTP)
{
- /* If this is a redirection, we must not allow recursive FTP
- retrieval, so we save recursion to oldrec, and restore it
- later. */
- int oldrec = opt.recursive;
+ /* If this is a redirection, temporarily turn off opt.ftp_glob
+ and opt.recursive, both being undesirable when following
+ redirects. */
+ bool oldrec = opt.recursive, oldglob = opt.ftp_glob;
if (redirection_count)
- opt.recursive = 0;
+ opt.recursive = opt.ftp_glob = false;
+
result = ftp_loop (u, dt, proxy_url);
opt.recursive = oldrec;
+ opt.ftp_glob = oldglob;
/* There is a possibility of having HTTP being redirected to
FTP. In these cases we must decide whether the text is HTML
return result;
}
-/* Find the URLs in the file and call retrieve_url() for each of
- them. If HTML is non-zero, treat the file as HTML, and construct
- the URLs accordingly.
+/* Find the URLs in the file and call retrieve_url() for each of them.
+ If HTML is true, treat the file as HTML, and construct the URLs
+ accordingly.
If opt.recursive is set, call retrieve_tree() for each file. */
uerr_t
-retrieve_from_file (const char *file, int html, int *count)
+retrieve_from_file (const char *file, bool html, int *count)
{
uerr_t status;
struct urlpos *url_list, *cur_url;
if (filename && opt.delete_after && file_exists_p (filename))
{
- DEBUGP (("Removing file due to --delete-after in"
- " retrieve_from_file():\n"));
+ DEBUGP (("\
+Removing file due to --delete-after in retrieve_from_file():\n"));
logprintf (LOG_VERBOSE, _("Removing %s.\n"), filename);
if (unlink (filename))
logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
void
sleep_between_retrievals (int count)
{
- static int first_retrieval = 1;
+ static bool first_retrieval = true;
if (first_retrieval)
{
/* Don't sleep before the very first retrieval. */
- first_retrieval = 0;
+ first_retrieval = false;
return;
}
rename(fname, to);
}
-static int no_proxy_match PARAMS ((const char *, const char **));
+static bool no_proxy_match (const char *, const char **);
/* Return the URL of the proxy appropriate for url U. */
}
/* Should a host be accessed through proxy, concerning no_proxy? */
-int
+static bool
no_proxy_match (const char *host, const char **no_proxy)
{
if (!no_proxy)
- return 1;
+ return true;
else
return !sufmatch (no_proxy, host);
}
projects / wget / blobdiff