[svn] New option --random-wait.

Submitted by Alan Eldridge in
<200111042106.fA4L63b75804@wwweasel.geeksrus.net>.

diff --git a/doc/ChangeLog b/doc/ChangeLog
index 901aa4b48bb09b74480575b90bedd566d61a0a32..6e9b2ac8ece24219470d3c27d3a7575aba52dd21 100644 (file)
--- a/doc/ChangeLog
+++ b/doc/ChangeLog
@@ -1,3 +1,7 @@
+2001-11-04  Alan Eldridge  <alane@geeksrus.net>
+
+       * wget.texi: Document --random-wait, randomwait=on/off.
+
 2001-11-23  Hrvoje Niksic  <hniksic@arsdigita.com>
 
        * wget.texi (Download Options): Document the new `--progress'

diff --git a/doc/wget.texi b/doc/wget.texi
index 7419c0d1a59eb7bd1eedfa80a79a3cc987e6f4d7..9b964fde258c3ff92ec4a19d433aed23b0f77ccd 100644 (file)
--- a/doc/wget.texi
+++ b/doc/wget.texi
@@ -701,6 +701,26 @@ seconds per file.
 Note that this option is turned on by default in the global
 @file{wgetrc} file.
 
+@cindex wait, random
+@cindex random wait
+@itemx --random-wait
+Some web sites may perform log analysis to identify retrieval programs
+such as Wget by looking for statistically significant similarities in
+the time between requests. This option causes the time between requests
+to vary between 0 and 2 * @var{wait} seconds, where @var{wait} was
+specified using the @samp{-w} or @samp{--wait} options, in order to mask
+Wget's presence from such analysis.
+
+A recent article in a publication devoted to development on a popular
+consumer platform provided code to perform this analysis on the fly.
+Its author suggested blocking at the class C address level to ensure
+automated retrieval programs were blocked despite changing DHCP-supplied
+addresses.
+
+The @samp{--random-wait} option was inspired by this ill-advised
+recommendation to block many unrelated users from a web site due to the
+actions of one.
+
 @cindex proxy
 @item -Y on/off
 @itemx --proxy=on/off
@@ -2168,6 +2188,10 @@ Wait @var{n} seconds between retrievals---the same as @samp{-w}.
 Wait up to @var{n} seconds between retries of failed retrievals
 only---the same as @samp{--waitretry}.  Note that this is turned on by
 default in the global @file{wgetrc}.
+
+@item randomwait = on/off
+Turn random between-request wait times on or off. The same as 
+@samp{--random-wait}.
 @end table
 
 @node Sample Wgetrc,  , Wgetrc Commands, Startup File

diff --git a/src/ChangeLog b/src/ChangeLog
index c7fdc75359872dfd2a79e982174e34a6788ab8b0..c051e0204821c1ae570557392b604c2a9124d605 100644 (file)
--- a/src/ChangeLog
+++ b/src/ChangeLog
@@ -1,3 +1,17 @@
+2001-11-04  Alan Eldridge  <alane@geeksrus.net>
+
+       * config.h.in: added HAVE_RANDOM.
+
+       * options.h: added random_wait to struct options.
+
+       * main.c (print_help [HAVE_RANDOM], main): added arg parsing, help
+       for --random-wait.
+
+       * retr.c (sleep_between_retrievals) [HAVE_RANDOM]: added
+       implementation of random wait times.
+
+       * init.c (commands): added "randomwait" keyword.
+
 2001-11-25  Hrvoje Niksic  <hniksic@arsdigita.com>
 
        * recur.c (descend_url_p): Be more conservative with blacklisting

diff --git a/src/init.c b/src/init.c
index f0a4f222fad9ad3160509ddedd4f93a16c272bb2..0d61b668139cf5def6f4cf9044c9fce09ec53e16 100644 (file)
--- a/src/init.c
+++ b/src/init.c
@@ -160,6 +160,7 @@ static struct {
   { "proxyuser",       &opt.proxy_user,        cmd_string },
   { "quiet",           &opt.quiet,             cmd_boolean },
   { "quota",           &opt.quota,             cmd_bytes },
+  { "randomwait",      &opt.random_wait,       cmd_boolean },
   { "reclevel",                &opt.reclevel,          cmd_number_inf },
   { "recursive",       NULL,                   cmd_spec_recursive },
   { "referer",         &opt.referer,           cmd_string },

diff --git a/src/main.c b/src/main.c
index f5cf13c20cf46193bb0b7fe9da8f5059740a77cb..0bdecf38696bc5a9c34b2e1a8c6844130fb8f77b 100644 (file)
--- a/src/main.c
+++ b/src/main.c
@@ -161,9 +161,14 @@ Download:\n\
   -T,  --timeout=SECONDS        set the read timeout to SECONDS.\n\
   -w,  --wait=SECONDS           wait SECONDS between retrievals.\n\
        --waitretry=SECONDS      wait 1...SECONDS between retries of a retrieval.\n\
+       --random-wait            wait from 0...2*WAIT secs between retrievals.\n\
   -Y,  --proxy=on/off           turn proxy on or off.\n\
   -Q,  --quota=NUMBER           set retrieval quota to NUMBER.\n\
 \n"), stdout);
+#ifdef HAVE_RANDOM
+  fputs (_("\
+\n"), stdout);
+#endif
   fputs (_("\
 Directories:\n\
   -nd  --no-directories            don\'t create directories.\n\
@@ -261,6 +266,7 @@ main (int argc, char *const *argv)
     { "passive-ftp", no_argument, NULL, 139 },
     { "page-requisites", no_argument, NULL, 'p' },
     { "quiet", no_argument, NULL, 'q' },
+    { "random-wait", no_argument, NULL, 165 },
     { "recursive", no_argument, NULL, 'r' },
     { "relative", no_argument, NULL, 'L' },
     { "retr-symlinks", no_argument, NULL, 137 },
@@ -395,6 +401,9 @@ hpVqvdkKsxmNWrHSLcFbEY:G:g:T:U:O:l:n:i:o:a:t:D:A:R:P:B:e:Q:X:I:w:C:",
        case 156:
          setval ("httpkeepalive", "off");
          break;
+       case 165:
+         setval ("randomwait", "on");
+         break;
        case 'b':
          setval ("background", "on");
          break;

diff --git a/src/options.h b/src/options.h
index 4f75c8b9d82cc8569266225682bd91fc33d52013..8ce4c6f86626c2bc10676a434de554c74bf81a0d 100644 (file)
--- a/src/options.h
+++ b/src/options.h
@@ -99,6 +99,7 @@ struct options
   long timeout;                        /* The value of read timeout in
                                   seconds. */
 #endif
+  int random_wait;             /* vary from 0 .. wait secs by random()? */
   long wait;                   /* The wait period between retrievals. */
   long waitretry;              /* The wait period between retries. - HEH */
   int use_robots;              /* Do we heed robots.txt? */

diff --git a/src/retr.c b/src/retr.c
index 93ef7db55f1e169ef18a3f93d6e5186519f4ba46..b8fb6c84b3a05e348a2784705294890438f7a670 100644 (file)
--- a/src/retr.c
+++ b/src/retr.c
@@ -560,8 +560,18 @@ sleep_between_retrievals (int count)
            sleep (opt.waitretry);
        }
       else if (opt.wait)
-       /* Otherwise, check if opt.wait is specified.  If so, sleep.  */
-       sleep (opt.wait);
+       {
+         /* Otherwise, check if opt.wait is specified.  If so, sleep.  */
+         if (count > 1 || !opt.random_wait)
+           sleep (opt.wait);
+         else
+           {
+             int waitsecs = random() % (opt.wait * 2 + 1);
+             DEBUGP(("sleep_between_retrievals: norm=%ld,random=%ld,sleep=%d\n",
+                     opt.wait, waitsecs - opt.wait, waitsecs));
+             sleep(waitsecs);
+           }
+       }
     }
   if (first_retrieval)
     first_retrieval = 0;