From bae3162c03e690ff566cdb0d9d1dee91cb27daee Mon Sep 17 00:00:00 2001 From: hniksic Date: Mon, 26 Nov 2001 12:07:13 -0800 Subject: [PATCH] [svn] New option `--limit-rate'. Published in . --- ChangeLog | 4 +++ configure.in | 2 +- src/ChangeLog | 19 ++++++++++ src/cmpt.c | 16 +++++++++ src/config.h.in | 3 ++ src/init.c | 1 + src/main.c | 5 +++ src/options.h | 2 ++ src/progress.c | 27 +++++++------- src/retr.c | 96 ++++++++++++++++++++++++++++++++++++++----------- src/sysdep.h | 3 ++ 11 files changed, 144 insertions(+), 34 deletions(-) diff --git a/ChangeLog b/ChangeLog index 1c1d5917..a3d0db08 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,7 @@ +2001-11-26 Hrvoje Niksic + + * configure.in: Check for usleep. + 2001-11-25 Hrvoje Niksic * util/dist-wget: New file: the script used for building Wget. diff --git a/configure.in b/configure.in index 079a3438..1cc995a2 100644 --- a/configure.in +++ b/configure.in @@ -175,7 +175,7 @@ AC_FUNC_MMAP AC_CHECK_FUNCS(strdup strstr strcasecmp strncasecmp strpbrk) AC_CHECK_FUNCS(gettimeofday mktime strptime) AC_CHECK_FUNCS(strerror snprintf vsnprintf select signal symlink access isatty) -AC_CHECK_FUNCS(uname gethostname) +AC_CHECK_FUNCS(uname gethostname usleep) dnl dnl Check if we need to compile in getopt.c. diff --git a/src/ChangeLog b/src/ChangeLog index c9245292..1bc8c0b8 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,3 +1,22 @@ +2001-11-26 Hrvoje Niksic + + * config.h.in: Put a HAVE_USLEEP stub. + + * cmpt.c (usleep): Replacement implementation of usleep using + select. + + * init.c: New option init_rate. + + * main.c (main): New option --limit-rate. + + * retr.c (limit_bandwidth): New function. + (get_contents): Call it to limit the bandwidth used when + downloading. + + * progress.c (dot_update): Would print the wrong download speed on + rows other than the first one when the download was continued. + (dot_finish): Ditto. + 2001-11-26 Ian Abbott * http.c (gethttp): fix undeclared variable 'err' when compiled diff --git a/src/cmpt.c b/src/cmpt.c index bfc7b2c9..7a7add09 100644 --- a/src/cmpt.c +++ b/src/cmpt.c @@ -1206,3 +1206,19 @@ strptime (buf, format, tm) return strptime_internal (buf, format, tm, &decided); } #endif /* not HAVE_STRPTIME */ + + +#ifndef HAVE_USLEEP +/* A simple usleep implementation based on select(). This will + probably not work on Windows. */ + +int +usleep (unsigned long usec) +{ + struct timeval tm; + tm.tv_sec = 0; + tm.tv_usec = usec; + select (0, NULL, NULL, NULL, &tm); + return 0; +} +#endif /* not HAVE_USLEEP */ diff --git a/src/config.h.in b/src/config.h.in index 189d614f..1364d27f 100644 --- a/src/config.h.in +++ b/src/config.h.in @@ -165,6 +165,9 @@ char *alloca (); /* Define if you have the gettext function. */ #undef HAVE_GETTEXT +/* Define if you have the usleep function. */ +#undef HAVE_USLEEP + /* Define if you have the header file. */ #undef HAVE_STRING_H diff --git a/src/init.c b/src/init.c index 38918f46..2b1db499 100644 --- a/src/init.c +++ b/src/init.c @@ -142,6 +142,7 @@ static struct { { "includedirectories", &opt.includes, cmd_directory_vector }, { "input", &opt.input_filename, cmd_file }, { "killlonger", &opt.kill_longer, cmd_boolean }, + { "limitrate", &opt.limit_rate, cmd_bytes }, { "loadcookies", &opt.cookies_input, cmd_file }, { "logfile", &opt.lfilename, cmd_file }, { "login", &opt.ftp_acc, cmd_string }, diff --git a/src/main.c b/src/main.c index 0bdecf38..eaf952e0 100644 --- a/src/main.c +++ b/src/main.c @@ -164,6 +164,7 @@ Download:\n\ --random-wait wait from 0...2*WAIT secs between retrievals.\n\ -Y, --proxy=on/off turn proxy on or off.\n\ -Q, --quota=NUMBER set retrieval quota to NUMBER.\n\ + --limit-rate=RATE limit download rate to RATE.\n\ \n"), stdout); #ifdef HAVE_RANDOM fputs (_("\ @@ -303,6 +304,7 @@ main (int argc, char *const *argv) { "include-directories", required_argument, NULL, 'I' }, { "input-file", required_argument, NULL, 'i' }, { "level", required_argument, NULL, 'l' }, + { "limit-rate", required_argument, NULL, 164 }, { "load-cookies", required_argument, NULL, 161 }, { "no", required_argument, NULL, 'n' }, { "output-document", required_argument, NULL, 'O' }, @@ -535,6 +537,9 @@ GNU General Public License for more details.\n")); case 163: setval ("progress", optarg); break; + case 164: + setval ("limitrate", optarg); + break; case 157: setval ("referer", optarg); break; diff --git a/src/options.h b/src/options.h index 8ce4c6f8..92b812a4 100644 --- a/src/options.h +++ b/src/options.h @@ -104,6 +104,8 @@ struct options long waitretry; /* The wait period between retries. - HEH */ int use_robots; /* Do we heed robots.txt? */ + long limit_rate; /* Limit the download rate to this + many bps. */ long quota; /* Maximum number of bytes to retrieve. */ VERY_LONG_TYPE downloaded; /* How much we downloaded already. */ diff --git a/src/progress.c b/src/progress.c index e010d2d3..7c9bba1e 100644 --- a/src/progress.c +++ b/src/progress.c @@ -195,8 +195,8 @@ dot_create (long initial, long total) /* Align the [ skipping ... ] line with the dots. To do that, insert the number of spaces equal to the number of digits in the skipped amount in K. */ - logprintf (LOG_VERBOSE, "\n%*s%s", - 2 + skipped_k_len, "", _("[ skipping %dK ]")); + logprintf (LOG_VERBOSE, _("\n%*s[ skipping %dK ]"), + 2 + skipped_k_len, "", skipped_k); } logprintf (LOG_VERBOSE, "\n%5ldK", skipped / 1024); @@ -255,15 +255,16 @@ dot_update (void *progress, long howmuch, long dltime) ++dp->dots; if (dp->dots >= opt.dots_in_line) { + long row_qty = row_bytes; + if (dp->rows == dp->initial_length / row_bytes) + row_qty -= dp->initial_length % row_bytes; + ++dp->rows; dp->dots = 0; if (dp->total_length) print_percentage (dp->rows * row_bytes, dp->total_length); - - print_download_speed (dp, - row_bytes - (dp->initial_length % row_bytes), - dltime); + print_download_speed (dp, row_qty, dltime); } } @@ -296,12 +297,14 @@ dot_finish (void *progress, long dltime) dp->total_length); } - print_download_speed (dp, dp->dots * dot_bytes - + dp->accumulated - - dp->initial_length % row_bytes, - dltime); - logputs (LOG_VERBOSE, "\n\n"); + { + long row_qty = dp->dots * dot_bytes + dp->accumulated; + if (dp->rows == dp->initial_length / row_bytes) + row_qty -= dp->initial_length % row_bytes; + print_download_speed (dp, row_qty, dltime); + } + logputs (LOG_VERBOSE, "\n\n"); log_set_flush (0); xfree (dp); @@ -401,7 +404,7 @@ bar_create (long initial, long total) bp->width = screen_width; bp->buffer = xmalloc (bp->width + 1); - logputs (LOG_VERBOSE, "\n\n"); + logputs (LOG_VERBOSE, "\n"); create_image (bp, 0); display_image (bp->buffer); diff --git a/src/retr.c b/src/retr.c index f4b80fdd..9930cc1e 100644 --- a/src/retr.c +++ b/src/retr.c @@ -52,6 +52,50 @@ extern int errno; int global_download_count; +static struct { + long bytes; + long dltime; +} limit_data; + +static void +limit_bandwidth_reset (void) +{ + limit_data.bytes = 0; + limit_data.dltime = 0; +} + +/* Limit the bandwidth by pausing the download for an amount of time. + BYTES is the number of bytes received from the network, DELTA is + how long it took to receive them, DLTIME the current download time, + TIMER the timer, and ADJUSTMENT the previous. */ + +static void +limit_bandwidth (long bytes, long delta) +{ + long expected; + + limit_data.bytes += bytes; + limit_data.dltime += delta; + + expected = (long)(1000.0 * limit_data.bytes / opt.limit_rate); + + if (expected > limit_data.dltime) + { + long slp = expected - limit_data.dltime; + if (slp < 200) + { + DEBUGP (("deferring a %ld ms sleep (%ld/%ld) until later.\n", + slp, limit_data.bytes, limit_data.dltime)); + return; + } + DEBUGP (("sleeping %ld ms\n", slp)); + usleep (1000 * slp); + } + + limit_data.bytes = 0; + limit_data.dltime = 0; +} + #define MIN(i, j) ((i) <= (j) ? (i) : (j)) /* Reads the contents of file descriptor FD, until it is closed, or a @@ -82,14 +126,13 @@ get_contents (int fd, FILE *fp, long *len, long restval, long expected, int res = 0; static char c[8192]; void *progress = NULL; - struct wget_timer *timer = NULL; + struct wget_timer *timer = wtimer_allocate (); + long dltime = 0, last_dltime = 0; *len = restval; if (opt.verbose) progress = progress_create (restval, expected); - if (opt.verbose || elapsed != NULL) - timer = wtimer_new (); if (rbuf && RBUF_FD (rbuf) == fd) { @@ -108,8 +151,13 @@ get_contents (int fd, FILE *fp, long *len, long restval, long expected, goto out; } if (opt.verbose) - progress_update (progress, sz, wtimer_elapsed (timer)); + progress_update (progress, sz, 0); } + + if (opt.limit_rate) + limit_bandwidth_reset (); + wtimer_reset (timer); + /* Read from fd while there is available data. Normally, if expected is 0, it means that it is not known how @@ -121,14 +169,12 @@ get_contents (int fd, FILE *fp, long *len, long restval, long expected, ? MIN (expected - *len, sizeof (c)) : sizeof (c)); #ifdef HAVE_SSL - if (rbuf->ssl!=NULL) { - res = ssl_iread (rbuf->ssl, c, amount_to_read); - } else { -#endif /* HAVE_SSL */ - res = iread (fd, c, amount_to_read); -#ifdef HAVE_SSL - } + if (rbuf->ssl!=NULL) + res = ssl_iread (rbuf->ssl, c, amount_to_read); + else #endif /* HAVE_SSL */ + res = iread (fd, c, amount_to_read); + if (res > 0) { fwrite (c, sizeof (char), res, fp); @@ -141,8 +187,19 @@ get_contents (int fd, FILE *fp, long *len, long restval, long expected, res = -2; goto out; } + + /* If bandwidth is not limited, one call to wtimer_elapsed + is sufficient. */ + dltime = wtimer_elapsed (timer); + if (opt.limit_rate) + { + limit_bandwidth (res, dltime - last_dltime); + dltime = wtimer_elapsed (timer); + last_dltime = dltime; + } + if (opt.verbose) - progress_update (progress, res, wtimer_elapsed (timer)); + progress_update (progress, res, dltime); *len += res; } else @@ -152,15 +209,12 @@ get_contents (int fd, FILE *fp, long *len, long restval, long expected, res = -1; out: - if (timer) - { - long dltime = wtimer_elapsed (timer); - if (opt.verbose) - progress_finish (progress, dltime); - if (elapsed) - *elapsed = dltime; - wtimer_delete (timer); - } + if (opt.verbose) + progress_finish (progress, dltime); + if (elapsed) + *elapsed = dltime; + wtimer_delete (timer); + return res; } diff --git a/src/sysdep.h b/src/sysdep.h index 74f8a7cf..1c5deb1b 100644 --- a/src/sysdep.h +++ b/src/sysdep.h @@ -163,6 +163,9 @@ char *strptime (); #ifndef HAVE_VSNPRINTF int vsnprintf (); #endif +#ifndef HAVE_USLEEP +int usleep (); +#endif /* SunOS brain damage -- for some reason, SunOS header files fail to declare the functions below, which causes all kinds of problems -- 2.39.2