#include "connect.h"
#include "hash.h"
+#ifdef HAVE_SSL
+# include "gen_sslfunc.h" /* for ssl_iread */
+#endif
+
#ifndef errno
extern int errno;
#endif
int global_download_count;
\f
+static struct {
+ long bytes;
+ long dltime;
+} limit_data;
+
+static void
+limit_bandwidth_reset (void)
+{
+ limit_data.bytes = 0;
+ limit_data.dltime = 0;
+}
+
+/* Limit the bandwidth by pausing the download for an amount of time.
+ BYTES is the number of bytes received from the network, DELTA is
+ how long it took to receive them, DLTIME the current download time,
+ TIMER the timer, and ADJUSTMENT the previous. */
+
+static void
+limit_bandwidth (long bytes, long delta)
+{
+ long expected;
+
+ limit_data.bytes += bytes;
+ limit_data.dltime += delta;
+
+ expected = (long)(1000.0 * limit_data.bytes / opt.limit_rate);
+
+ if (expected > limit_data.dltime)
+ {
+ long slp = expected - limit_data.dltime;
+ if (slp < 200)
+ {
+ DEBUGP (("deferring a %ld ms sleep (%ld/%ld) until later.\n",
+ slp, limit_data.bytes, limit_data.dltime));
+ return;
+ }
+ DEBUGP (("sleeping %ld ms\n", slp));
+ usleep (1000 * slp);
+ }
+
+ limit_data.bytes = 0;
+ limit_data.dltime = 0;
+}
+
#define MIN(i, j) ((i) <= (j) ? (i) : (j))
/* Reads the contents of file descriptor FD, until it is closed, or a
from fd immediately, flush or discard the buffer. */
int
get_contents (int fd, FILE *fp, long *len, long restval, long expected,
- struct rbuf *rbuf, int use_expected)
+ struct rbuf *rbuf, int use_expected, long *elapsed)
{
int res = 0;
static char c[8192];
void *progress = NULL;
+ struct wget_timer *timer = wtimer_allocate ();
+ long dltime = 0, last_dltime = 0;
*len = restval;
+
if (opt.verbose)
progress = progress_create (restval, expected);
if (rbuf && RBUF_FD (rbuf) == fd)
{
- int need_flush = 0;
+ int sz = 0;
while ((res = rbuf_flush (rbuf, c, sizeof (c))) != 0)
{
- if (fwrite (c, sizeof (char), res, fp) < res)
- return -2;
- if (opt.verbose)
- progress_update (progress, res);
+ fwrite (c, sizeof (char), res, fp);
*len += res;
- need_flush = 1;
+ sz += res;
}
- if (need_flush)
+ if (sz)
fflush (fp);
if (ferror (fp))
- return -2;
+ {
+ res = -2;
+ goto out;
+ }
+ if (opt.verbose)
+ progress_update (progress, sz, 0);
}
+
+ if (opt.limit_rate)
+ limit_bandwidth_reset ();
+ wtimer_reset (timer);
+
/* Read from fd while there is available data.
Normally, if expected is 0, it means that it is not known how
? MIN (expected - *len, sizeof (c))
: sizeof (c));
#ifdef HAVE_SSL
- if (rbuf->ssl!=NULL) {
- res = ssl_iread (rbuf->ssl, c, amount_to_read);
- } else {
-#endif /* HAVE_SSL */
- res = iread (fd, c, amount_to_read);
-#ifdef HAVE_SSL
- }
+ if (rbuf->ssl!=NULL)
+ res = ssl_iread (rbuf->ssl, c, amount_to_read);
+ else
#endif /* HAVE_SSL */
+ res = iread (fd, c, amount_to_read);
+
if (res > 0)
{
fwrite (c, sizeof (char), res, fp);
packets typically won't be too tiny anyway. */
fflush (fp);
if (ferror (fp))
- return -2;
+ {
+ res = -2;
+ goto out;
+ }
+
+ /* If bandwidth is not limited, one call to wtimer_elapsed
+ is sufficient. */
+ dltime = wtimer_elapsed (timer);
+ if (opt.limit_rate)
+ {
+ limit_bandwidth (res, dltime - last_dltime);
+ dltime = wtimer_elapsed (timer);
+ last_dltime = dltime;
+ }
+
if (opt.verbose)
- progress_update (progress, res);
+ progress_update (progress, res, dltime);
*len += res;
}
else
}
if (res < -1)
res = -1;
+
+ out:
if (opt.verbose)
- progress_finish (progress);
+ progress_finish (progress, dltime);
+ if (elapsed)
+ *elapsed = dltime;
+ wtimer_delete (timer);
+
return res;
}
\f
uerr_t status;
struct urlpos *url_list, *cur_url;
- url_list = (html ? get_urls_html (file, NULL, FALSE, NULL)
+ url_list = (html ? get_urls_html (file, NULL, NULL)
: get_urls_file (file));
status = RETROK; /* Suppose everything is OK. */
*count = 0; /* Reset the URL count. */
for (cur_url = url_list; cur_url; cur_url = cur_url->next, ++*count)
{
- char *filename = NULL, *new_file;
+ char *filename = NULL, *new_file = NULL;
int dt;
if (cur_url->ignore_when_downloading)
{
static int first_retrieval = 1;
+ if (first_retrieval && opt.random_wait)
+ /* --random-wait uses the RNG, so seed it. */
+ srand (time (NULL));
+
if (!first_retrieval && (opt.wait || opt.waitretry))
{
if (opt.waitretry && count > 1)
sleep (opt.wait);
else
{
- int waitsecs = random() % (opt.wait * 2 + 1);
- DEBUGP(("sleep_between_retrievals: norm=%ld,random=%ld,sleep=%d\n",
- opt.wait, waitsecs - opt.wait, waitsecs));
- sleep(waitsecs);
+ int waitmax = 2 * opt.wait;
+ /* This is equivalent to rand() % waitmax, but uses the
+ high-order bits for better randomness. */
+ int waitsecs = (double)waitmax * rand () / (RAND_MAX + 1.0);
+
+ DEBUGP (("sleep_between_retrievals: norm=%ld,fuzz=%ld,sleep=%d\n",
+ opt.wait, waitsecs - opt.wait, waitsecs));
+
+ if (waitsecs)
+ sleep (waitsecs);
}
}
}
projects / wget / blobdiff