[svn] New option `--limit-rate'.

Published in <sxsd725cn56.fsf@florida.arsdigita.de>.

diff --git a/ChangeLog b/ChangeLog
index 1c1d591763ab242f5fab98eb09ef0ac92b504cb6..a3d0db08faf9f86c67f1a16c127810e18c0003a4 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,7 @@
+2001-11-26  Hrvoje Niksic  <hniksic@arsdigita.com>
+
+       * configure.in: Check for usleep.
+
 2001-11-25  Hrvoje Niksic  <hniksic@arsdigita.com>
 
        * util/dist-wget: New file: the script used for building Wget.

diff --git a/configure.in b/configure.in
index 079a343873d53ecb5712bea42c878f98563e3c26..1cc995a2000e24440fb7b7aa18c34e6d9c478744 100644 (file)
--- a/configure.in
+++ b/configure.in
@@ -175,7 +175,7 @@ AC_FUNC_MMAP
 AC_CHECK_FUNCS(strdup strstr strcasecmp strncasecmp strpbrk)
 AC_CHECK_FUNCS(gettimeofday mktime strptime)
 AC_CHECK_FUNCS(strerror snprintf vsnprintf select signal symlink access isatty)
-AC_CHECK_FUNCS(uname gethostname)
+AC_CHECK_FUNCS(uname gethostname usleep)
 
 dnl
 dnl Check if we need to compile in getopt.c.

diff --git a/src/ChangeLog b/src/ChangeLog
index c92452922c389bcc236de5b14d94d2b55d02c9b2..1bc8c0b857525f6e3fdee7345f307699d5a1acf6 100644 (file)
--- a/src/ChangeLog
+++ b/src/ChangeLog
@@ -1,3 +1,22 @@
+2001-11-26  Hrvoje Niksic  <hniksic@arsdigita.com>
+
+       * config.h.in: Put a HAVE_USLEEP stub.
+
+       * cmpt.c (usleep): Replacement implementation of usleep using
+       select.
+
+       * init.c: New option init_rate.
+
+       * main.c (main): New option --limit-rate.
+
+       * retr.c (limit_bandwidth): New function.
+       (get_contents): Call it to limit the bandwidth used when
+       downloading.
+
+       * progress.c (dot_update): Would print the wrong download speed on
+       rows other than the first one when the download was continued.
+       (dot_finish): Ditto.
+
 2001-11-26  Ian Abbott <abbotti@mev.co.uk>
 
        * http.c (gethttp): fix undeclared variable 'err' when compiled

diff --git a/src/cmpt.c b/src/cmpt.c
index bfc7b2c9ab06275ab043062187f5becd594fba43..7a7add09bd607da57a3b908c5c73b3e1d8ea9d59 100644 (file)
--- a/src/cmpt.c
+++ b/src/cmpt.c
@@ -1206,3 +1206,19 @@ strptime (buf, format, tm)
   return strptime_internal (buf, format, tm, &decided);
 }
 #endif /* not HAVE_STRPTIME */
+
+
+#ifndef HAVE_USLEEP
+/* A simple usleep implementation based on select().  This will
+   probably not work on Windows.  */
+
+int
+usleep (unsigned long usec)
+{
+  struct timeval tm;
+  tm.tv_sec = 0;
+  tm.tv_usec = usec;
+  select (0, NULL, NULL, NULL, &tm);
+  return 0;
+}
+#endif /* not HAVE_USLEEP */

diff --git a/src/config.h.in b/src/config.h.in
index 189d614f8b7127b85579ab9993ff6d670e7f702f..1364d27f7fd54efcabe5afbde8d4d3295e38fc40 100644 (file)
--- a/src/config.h.in
+++ b/src/config.h.in
@@ -165,6 +165,9 @@ char *alloca ();
 /* Define if you have the gettext function.  */
 #undef HAVE_GETTEXT
 
+/* Define if you have the usleep function.  */
+#undef HAVE_USLEEP
+
 /* Define if you have the <string.h> header file.  */
 #undef HAVE_STRING_H
 

diff --git a/src/init.c b/src/init.c
index 38918f4643a7b0ee4a66b796150d2093bc5753f1..2b1db499f08cfa6bd2250964e1c9556697996916 100644 (file)
--- a/src/init.c
+++ b/src/init.c
@@ -142,6 +142,7 @@ static struct {
   { "includedirectories", &opt.includes,       cmd_directory_vector },
   { "input",           &opt.input_filename,    cmd_file },
   { "killlonger",      &opt.kill_longer,       cmd_boolean },
+  { "limitrate",       &opt.limit_rate,        cmd_bytes },
   { "loadcookies",     &opt.cookies_input,     cmd_file },
   { "logfile",         &opt.lfilename,         cmd_file },
   { "login",           &opt.ftp_acc,           cmd_string },

diff --git a/src/main.c b/src/main.c
index 0bdecf38696bc5a9c34b2e1a8c6844130fb8f77b..eaf952e0850c70829da7acdd70495947680b3f0f 100644 (file)
--- a/src/main.c
+++ b/src/main.c
@@ -164,6 +164,7 @@ Download:\n\
        --random-wait            wait from 0...2*WAIT secs between retrievals.\n\
   -Y,  --proxy=on/off           turn proxy on or off.\n\
   -Q,  --quota=NUMBER           set retrieval quota to NUMBER.\n\
+       --limit-rate=RATE        limit download rate to RATE.\n\
 \n"), stdout);
 #ifdef HAVE_RANDOM
   fputs (_("\
@@ -303,6 +304,7 @@ main (int argc, char *const *argv)
     { "include-directories", required_argument, NULL, 'I' },
     { "input-file", required_argument, NULL, 'i' },
     { "level", required_argument, NULL, 'l' },
+    { "limit-rate", required_argument, NULL, 164 },
     { "load-cookies", required_argument, NULL, 161 },
     { "no", required_argument, NULL, 'n' },
     { "output-document", required_argument, NULL, 'O' },
@@ -535,6 +537,9 @@ GNU General Public License for more details.\n"));
        case 163:
          setval ("progress", optarg);
          break;
+       case 164:
+         setval ("limitrate", optarg);
+         break;
        case 157:
          setval ("referer", optarg);
          break;

diff --git a/src/options.h b/src/options.h
index 8ce4c6f86626c2bc10676a434de554c74bf81a0d..92b812a4edb8cab41ba4b8e5b1ce2eb14743c04c 100644 (file)
--- a/src/options.h
+++ b/src/options.h
@@ -104,6 +104,8 @@ struct options
   long waitretry;              /* The wait period between retries. - HEH */
   int use_robots;              /* Do we heed robots.txt? */
 
+  long limit_rate;             /* Limit the download rate to this
+                                  many bps. */
   long quota;                  /* Maximum number of bytes to
                                   retrieve. */
   VERY_LONG_TYPE downloaded;   /* How much we downloaded already. */

diff --git a/src/progress.c b/src/progress.c
index e010d2d34ee6662219f931bfaf2e2013a534cfbd..7c9bba1e3e654b5027b1be7199bcd10c5eaa5b57 100644 (file)
--- a/src/progress.c
+++ b/src/progress.c
@@ -195,8 +195,8 @@ dot_create (long initial, long total)
          /* Align the [ skipping ... ] line with the dots.  To do
             that, insert the number of spaces equal to the number of
             digits in the skipped amount in K.  */
-         logprintf (LOG_VERBOSE, "\n%*s%s",
-                    2 + skipped_k_len, "", _("[ skipping %dK ]"));
+         logprintf (LOG_VERBOSE, _("\n%*s[ skipping %dK ]"),
+                    2 + skipped_k_len, "", skipped_k);
        }
 
       logprintf (LOG_VERBOSE, "\n%5ldK", skipped / 1024);
@@ -255,15 +255,16 @@ dot_update (void *progress, long howmuch, long dltime)
       ++dp->dots;
       if (dp->dots >= opt.dots_in_line)
        {
+         long row_qty = row_bytes;
+         if (dp->rows == dp->initial_length / row_bytes)
+           row_qty -= dp->initial_length % row_bytes;
+
          ++dp->rows;
          dp->dots = 0;
 
          if (dp->total_length)
            print_percentage (dp->rows * row_bytes, dp->total_length);
-
-         print_download_speed (dp,
-                               row_bytes - (dp->initial_length % row_bytes),
-                               dltime);
+         print_download_speed (dp, row_qty, dltime);
        }
     }
 
@@ -296,12 +297,14 @@ dot_finish (void *progress, long dltime)
                        dp->total_length);
     }
 
-  print_download_speed (dp, dp->dots * dot_bytes
-                       + dp->accumulated
-                       - dp->initial_length % row_bytes,
-                       dltime);
-  logputs (LOG_VERBOSE, "\n\n");
+  {
+    long row_qty = dp->dots * dot_bytes + dp->accumulated;
+    if (dp->rows == dp->initial_length / row_bytes)
+      row_qty -= dp->initial_length % row_bytes;
+    print_download_speed (dp, row_qty, dltime);
+  }
 
+  logputs (LOG_VERBOSE, "\n\n");
   log_set_flush (0);
 
   xfree (dp);
@@ -401,7 +404,7 @@ bar_create (long initial, long total)
   bp->width = screen_width;
   bp->buffer = xmalloc (bp->width + 1);
 
-  logputs (LOG_VERBOSE, "\n\n");
+  logputs (LOG_VERBOSE, "\n");
 
   create_image (bp, 0);
   display_image (bp->buffer);

diff --git a/src/retr.c b/src/retr.c
index f4b80fdd50b609f8289653c7a8ee2b073403ca81..9930cc1e0f4782bf27eea814672329ca344e2696 100644 (file)
--- a/src/retr.c
+++ b/src/retr.c
@@ -52,6 +52,50 @@ extern int errno;
 int global_download_count;
 
 \f
+static struct {
+  long bytes;
+  long dltime;
+} limit_data;
+
+static void
+limit_bandwidth_reset (void)
+{
+  limit_data.bytes  = 0;
+  limit_data.dltime = 0;
+}
+
+/* Limit the bandwidth by pausing the download for an amount of time.
+   BYTES is the number of bytes received from the network, DELTA is
+   how long it took to receive them, DLTIME the current download time,
+   TIMER the timer, and ADJUSTMENT the previous.  */
+
+static void
+limit_bandwidth (long bytes, long delta)
+{
+  long expected;
+
+  limit_data.bytes += bytes;
+  limit_data.dltime += delta;
+
+  expected = (long)(1000.0 * limit_data.bytes / opt.limit_rate);
+
+  if (expected > limit_data.dltime)
+    {
+      long slp = expected - limit_data.dltime;
+      if (slp < 200)
+       {
+         DEBUGP (("deferring a %ld ms sleep (%ld/%ld) until later.\n",
+                  slp, limit_data.bytes, limit_data.dltime));
+         return;
+       }
+      DEBUGP (("sleeping %ld ms\n", slp));
+      usleep (1000 * slp);
+    }
+
+  limit_data.bytes = 0;
+  limit_data.dltime = 0;
+}
+
 #define MIN(i, j) ((i) <= (j) ? (i) : (j))
 
 /* Reads the contents of file descriptor FD, until it is closed, or a
@@ -82,14 +126,13 @@ get_contents (int fd, FILE *fp, long *len, long restval, long expected,
   int res = 0;
   static char c[8192];
   void *progress = NULL;
-  struct wget_timer *timer = NULL;
+  struct wget_timer *timer = wtimer_allocate ();
+  long dltime = 0, last_dltime = 0;
 
   *len = restval;
 
   if (opt.verbose)
     progress = progress_create (restval, expected);
-  if (opt.verbose || elapsed != NULL)
-    timer = wtimer_new ();
 
   if (rbuf && RBUF_FD (rbuf) == fd)
     {
@@ -108,8 +151,13 @@ get_contents (int fd, FILE *fp, long *len, long restval, long expected,
          goto out;
        }
       if (opt.verbose)
-       progress_update (progress, sz, wtimer_elapsed (timer));
+       progress_update (progress, sz, 0);
     }
+
+  if (opt.limit_rate)
+    limit_bandwidth_reset ();
+  wtimer_reset (timer);
+
   /* Read from fd while there is available data.
 
      Normally, if expected is 0, it means that it is not known how
@@ -121,14 +169,12 @@ get_contents (int fd, FILE *fp, long *len, long restval, long expected,
                            ? MIN (expected - *len, sizeof (c))
                            : sizeof (c));
 #ifdef HAVE_SSL
-               if (rbuf->ssl!=NULL) {
-                 res = ssl_iread (rbuf->ssl, c, amount_to_read);
-               } else {
-#endif /* HAVE_SSL */
-                 res = iread (fd, c, amount_to_read);
-#ifdef HAVE_SSL
-               }
+      if (rbuf->ssl!=NULL)
+       res = ssl_iread (rbuf->ssl, c, amount_to_read);
+      else
 #endif /* HAVE_SSL */
+       res = iread (fd, c, amount_to_read);
+
       if (res > 0)
        {
          fwrite (c, sizeof (char), res, fp);
@@ -141,8 +187,19 @@ get_contents (int fd, FILE *fp, long *len, long restval, long expected,
              res = -2;
              goto out;
            }
+
+         /* If bandwidth is not limited, one call to wtimer_elapsed
+            is sufficient.  */
+         dltime = wtimer_elapsed (timer);
+         if (opt.limit_rate)
+           {
+             limit_bandwidth (res, dltime - last_dltime);
+             dltime = wtimer_elapsed (timer);
+             last_dltime = dltime;
+           }
+
          if (opt.verbose)
-           progress_update (progress, res, wtimer_elapsed (timer));
+           progress_update (progress, res, dltime);
          *len += res;
        }
       else
@@ -152,15 +209,12 @@ get_contents (int fd, FILE *fp, long *len, long restval, long expected,
     res = -1;
 
  out:
-  if (timer)
-    {
-      long dltime = wtimer_elapsed (timer);
-      if (opt.verbose)
-       progress_finish (progress, dltime);
-      if (elapsed)
-       *elapsed = dltime;
-      wtimer_delete (timer);
-    }
+  if (opt.verbose)
+    progress_finish (progress, dltime);
+  if (elapsed)
+    *elapsed = dltime;
+  wtimer_delete (timer);
+
   return res;
 }
 \f

diff --git a/src/sysdep.h b/src/sysdep.h
index 74f8a7cfe5ede531259530e1bad337f63ba24772..1c5deb1b8290c293e59b69a6b3da24b571ecd77e 100644 (file)
--- a/src/sysdep.h
+++ b/src/sysdep.h
@@ -163,6 +163,9 @@ char *strptime ();
 #ifndef HAVE_VSNPRINTF
 int vsnprintf ();
 #endif
+#ifndef HAVE_USLEEP
+int usleep ();
+#endif
 
 /* SunOS brain damage -- for some reason, SunOS header files fail to
    declare the functions below, which causes all kinds of problems