[svn] New option --retry-connrefused from Ahmod Dancy.

[wget] / src / main.c

diff --git a/src/main.c b/src/main.c
index 79e17af05bf729de694a39c9cba614e11ae31fd2..cfcdbbe3ad52f9e72fefbbaa31fb4505231d62a7 100644 (file)
--- a/src/main.c
+++ b/src/main.c
@@ -1,5 +1,5 @@
 /* Command line parsing.
 /* Command line parsing.

-   Copyright (C) 1995, 1996, 1997, 1998, 2000, 2001
+   Copyright (C) 1995, 1996, 1997, 1998, 2000, 2001, 2002

    Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
    Free Software Foundation, Inc.
 
 This file is part of GNU Wget.


@@ -16,7 +16,17 @@ GNU General Public License for more details.
 
 You should have received a copy of the GNU General Public License
 along with Wget; if not, write to the Free Software
 
 You should have received a copy of the GNU General Public License
 along with Wget; if not, write to the Free Software

-Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.  */
+Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+In addition, as a special exception, the Free Software Foundation
+gives permission to link the code of its release of Wget with the
+OpenSSL project's "OpenSSL" library (or with modified versions of it
+that use the same license as the "OpenSSL" library), and distribute
+the linked executables.  You must obey the GNU General Public License
+in all respects for all of the code used other than "OpenSSL".  If you
+modify this file, you may extend this exception to your version of the
+file, but you are not obligated to do so.  If you do not wish to do
+so, delete this exception statement from your version.  */

 
 #include <config.h>
 
 
 #include <config.h>
 


@@ -41,8 +51,6 @@ Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.  */
 #endif /* HAVE_NLS */
 #include <errno.h>
 
 #endif /* HAVE_NLS */
 #include <errno.h>
 

-#define OPTIONS_DEFINED_HERE   /* for options.h */
-

 #include "wget.h"
 #include "utils.h"
 #include "init.h"
 #include "wget.h"
 #include "utils.h"
 #include "init.h"


@@ -53,6 +61,10 @@ Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.  */
 #include "url.h"
 #include "progress.h"          /* for progress_handle_sigwinch */
 
 #include "url.h"
 #include "progress.h"          /* for progress_handle_sigwinch */
 

+#ifdef HAVE_SSL
+# include "gen_sslfunc.h"
+#endif
+

 /* On GNU system this will include system-wide getopt.h. */
 #include "getopt.h"
 
 /* On GNU system this will include system-wide getopt.h. */
 #include "getopt.h"
 


@@ -68,10 +80,12 @@ extern int errno;
 
 struct options opt;
 
 
 struct options opt;
 

+extern struct cookie_jar *wget_cookie_jar;
+

 /* From log.c.  */
 void log_init PARAMS ((const char *, int));
 void log_close PARAMS ((void));
 /* From log.c.  */
 void log_init PARAMS ((const char *, int));
 void log_close PARAMS ((void));

-void redirect_output PARAMS ((const char *));
+void log_request_redirect_output PARAMS ((const char *));

 
 static RETSIGTYPE redirect_output_signal PARAMS ((int));
 
 
 static RETSIGTYPE redirect_output_signal PARAMS ((int));
 


@@ -104,18 +118,6 @@ i18n_initialize (void)
   textdomain ("wget");
 #endif /* HAVE_NLS */
 }
   textdomain ("wget");
 #endif /* HAVE_NLS */
 }

-
-/* It's kosher to declare these here because their interface _has_ to
-   be void foo(void).  */
-void host_init PARAMS ((void));
-
-/* This just calls the various initialization functions from the
-   modules that need one-time initialization. */
-static void
-private_initialize (void)
-{
-  host_init ();
-}

 \f
 /* Print the usage message.  */
 static void
 \f
 /* Print the usage message.  */
 static void


@@ -156,13 +158,11 @@ Logging and input file:\n\
   -i,  --input-file=FILE      download URLs found in FILE.\n\
   -F,  --force-html           treat input file as HTML.\n\
   -B,  --base=URL             prepends URL to relative links in -F -i file.\n\
   -i,  --input-file=FILE      download URLs found in FILE.\n\
   -F,  --force-html           treat input file as HTML.\n\
   -B,  --base=URL             prepends URL to relative links in -F -i file.\n\

-       --sslcertfile=FILE     optional client certificate.\n\
-       --sslcertkey=KEYFILE   optional keyfile for this certificate.\n\
-\n"), stdout);
+\n"),stdout);

   fputs (_("\
 Download:\n\
   fputs (_("\
 Download:\n\

-       --bind-address=ADDRESS   bind to ADDRESS (hostname or IP) on local host.\n\

   -t,  --tries=NUMBER           set number of retries to NUMBER (0 unlimits).\n\
   -t,  --tries=NUMBER           set number of retries to NUMBER (0 unlimits).\n\

+       --retry-connrefused      retry even if connection is refused.\n\

   -O   --output-document=FILE   write documents to FILE.\n\
   -nc, --no-clobber             don\'t clobber existing files or use .# suffixes.\n\
   -c,  --continue               resume getting a partially-downloaded file.\n\
   -O   --output-document=FILE   write documents to FILE.\n\
   -nc, --no-clobber             don\'t clobber existing files or use .# suffixes.\n\
   -c,  --continue               resume getting a partially-downloaded file.\n\


@@ -173,8 +173,11 @@ Download:\n\
   -T,  --timeout=SECONDS        set the read timeout to SECONDS.\n\
   -w,  --wait=SECONDS           wait SECONDS between retrievals.\n\
        --waitretry=SECONDS      wait 1...SECONDS between retries of a retrieval.\n\
   -T,  --timeout=SECONDS        set the read timeout to SECONDS.\n\
   -w,  --wait=SECONDS           wait SECONDS between retrievals.\n\
        --waitretry=SECONDS      wait 1...SECONDS between retries of a retrieval.\n\

+       --random-wait            wait from 0...2*WAIT secs between retrievals.\n\

   -Y,  --proxy=on/off           turn proxy on or off.\n\
   -Q,  --quota=NUMBER           set retrieval quota to NUMBER.\n\
   -Y,  --proxy=on/off           turn proxy on or off.\n\
   -Q,  --quota=NUMBER           set retrieval quota to NUMBER.\n\

+       --bind-address=ADDRESS   bind to ADDRESS (hostname or IP) on local host.\n\
+       --limit-rate=RATE        limit download rate to RATE.\n\

 \n"), stdout);
   fputs (_("\
 Directories:\n\
 \n"), stdout);
   fputs (_("\
 Directories:\n\


@@ -201,7 +204,23 @@ HTTP options:\n\
        --cookies=off         don't use cookies.\n\
        --load-cookies=FILE   load cookies from FILE before session.\n\
        --save-cookies=FILE   save cookies to FILE after session.\n\
        --cookies=off         don't use cookies.\n\
        --load-cookies=FILE   load cookies from FILE before session.\n\
        --save-cookies=FILE   save cookies to FILE after session.\n\

+       --post-data=STRING    use the POST method; send STRING as the data.\n\
+       --post-file=FILE      use the POST method; send contents of FILE.\n\
+\n"), stdout);
+#ifdef HAVE_SSL
+  fputs (_("\
+HTTPS (SSL) options:\n\
+       --sslcertfile=FILE     optional client certificate.\n\
+       --sslcertkey=KEYFILE   optional keyfile for this certificate.\n\
+       --egd-file=FILE        file name of the EGD socket.\n\
+       --sslcadir=DIR         dir where hash list of CA's are stured.\n\
+       --sslcafile=FILE       file with bundle of CA's\n\
+       --sslcerttype=0/1      Client-Cert type 0=PEM (default) / 1=ASN1 (DER)\n\
+       --sslcheckcert=0/1     Check the server cert agenst given CA\n\
+       --sslprotocol=0-3      choose SSL protocol; 0=automatic,\n\
+                              1=SSLv2 2=SSLv3 3=TLSv1\n\

 \n"), stdout);
 \n"), stdout);

+#endif

   fputs (_("\
 FTP options:\n\
   -nr, --dont-remove-listing   don\'t remove `.listing\' files.\n\
   fputs (_("\
 FTP options:\n\
   -nr, --dont-remove-listing   don\'t remove `.listing\' files.\n\


@@ -211,7 +230,7 @@ FTP options:\n\
 \n"), stdout);
   fputs (_("\
 Recursive retrieval:\n\
 \n"), stdout);
   fputs (_("\
 Recursive retrieval:\n\

-  -r,  --recursive          recursive web-suck -- use with care!\n\
+  -r,  --recursive          recursive download.\n\

   -l,  --level=NUMBER       maximum recursion depth (inf or 0 for infinite).\n\
        --delete-after       delete files locally after downloading them.\n\
   -k,  --convert-links      convert non-relative links to relative.\n\
   -l,  --level=NUMBER       maximum recursion depth (inf or 0 for infinite).\n\
        --delete-after       delete files locally after downloading them.\n\
   -k,  --convert-links      convert non-relative links to relative.\n\


@@ -232,7 +251,6 @@ Recursive accept/reject:\n\
   -L,  --relative                   follow relative links only.\n\
   -I,  --include-directories=LIST   list of allowed directories.\n\
   -X,  --exclude-directories=LIST   list of excluded directories.\n\
   -L,  --relative                   follow relative links only.\n\
   -I,  --include-directories=LIST   list of allowed directories.\n\
   -X,  --exclude-directories=LIST   list of excluded directories.\n\

-  -nh, --no-host-lookup             don\'t DNS-lookup hosts.\n\

   -np, --no-parent                  don\'t ascend to the parent directory.\n\
 \n"), stdout);
   fputs (_("Mail bug reports and suggestions to <bug-wget@gnu.org>.\n"),
   -np, --no-parent                  don\'t ascend to the parent directory.\n\
 \n"), stdout);
   fputs (_("Mail bug reports and suggestions to <bug-wget@gnu.org>.\n"),


@@ -273,9 +291,11 @@ main (int argc, char *const *argv)
     { "passive-ftp", no_argument, NULL, 139 },
     { "page-requisites", no_argument, NULL, 'p' },
     { "quiet", no_argument, NULL, 'q' },
     { "passive-ftp", no_argument, NULL, 139 },
     { "page-requisites", no_argument, NULL, 'p' },
     { "quiet", no_argument, NULL, 'q' },

+    { "random-wait", no_argument, NULL, 165 },

     { "recursive", no_argument, NULL, 'r' },
     { "relative", no_argument, NULL, 'L' },
     { "retr-symlinks", no_argument, NULL, 137 },
     { "recursive", no_argument, NULL, 'r' },
     { "relative", no_argument, NULL, 'L' },
     { "retr-symlinks", no_argument, NULL, 137 },

+    { "retry-connrefused", no_argument, NULL, 174 },

     { "save-headers", no_argument, NULL, 's' },
     { "server-response", no_argument, NULL, 'S' },
     { "span-hosts", no_argument, NULL, 'H' },
     { "save-headers", no_argument, NULL, 's' },
     { "server-response", no_argument, NULL, 'S' },
     { "span-hosts", no_argument, NULL, 'H' },


@@ -309,10 +329,13 @@ main (int argc, char *const *argv)
     { "include-directories", required_argument, NULL, 'I' },
     { "input-file", required_argument, NULL, 'i' },
     { "level", required_argument, NULL, 'l' },
     { "include-directories", required_argument, NULL, 'I' },
     { "input-file", required_argument, NULL, 'i' },
     { "level", required_argument, NULL, 'l' },

+    { "limit-rate", required_argument, NULL, 164 },

     { "load-cookies", required_argument, NULL, 161 },
     { "no", required_argument, NULL, 'n' },
     { "output-document", required_argument, NULL, 'O' },
     { "output-file", required_argument, NULL, 'o' },
     { "load-cookies", required_argument, NULL, 161 },
     { "no", required_argument, NULL, 'n' },
     { "output-document", required_argument, NULL, 'O' },
     { "output-file", required_argument, NULL, 'o' },

+    { "post-data", required_argument, NULL, 167 },
+    { "post-file", required_argument, NULL, 168 },

     { "progress", required_argument, NULL, 163 },
     { "proxy", required_argument, NULL, 'Y' },
     { "proxy-passwd", required_argument, NULL, 144 },
     { "progress", required_argument, NULL, 163 },
     { "proxy", required_argument, NULL, 'Y' },
     { "proxy-passwd", required_argument, NULL, 144 },


@@ -328,6 +351,12 @@ main (int argc, char *const *argv)
 #ifdef HAVE_SSL
     { "sslcertfile", required_argument, NULL, 158 },
     { "sslcertkey", required_argument, NULL, 159 },
 #ifdef HAVE_SSL
     { "sslcertfile", required_argument, NULL, 158 },
     { "sslcertkey", required_argument, NULL, 159 },

+    { "egd-file", required_argument, NULL, 166 },
+    { "sslcadir",         required_argument, NULL, 169},
+    { "sslcafile",        required_argument, NULL, 170},
+    { "sslcerttype",      required_argument, NULL, 171},
+    { "sslcheckcert",     required_argument, NULL, 172},
+    { "sslprotocol",      required_argument, NULL, 173},

 #endif /* HAVE_SSL */
     { "wait", required_argument, NULL, 'w' },
     { "waitretry", required_argument, NULL, 152 },
 #endif /* HAVE_SSL */
     { "wait", required_argument, NULL, 'w' },
     { "waitretry", required_argument, NULL, 152 },


@@ -335,7 +364,6 @@ main (int argc, char *const *argv)
   };
 
   i18n_initialize ();
   };
 
   i18n_initialize ();

-  private_initialize ();

 
   append_to_log = 0;
 
 
   append_to_log = 0;
 


@@ -408,6 +436,9 @@ hpVqvdkKsxmNWrHSLcFbEY:G:g:T:U:O:l:n:i:o:a:t:D:A:R:P:B:e:Q:X:I:w:C:",
        case 156:
          setval ("httpkeepalive", "off");
          break;
        case 156:
          setval ("httpkeepalive", "off");
          break;

+       case 165:
+         setval ("randomwait", "on");
+         break;

        case 'b':
          setval ("background", "on");
          break;
        case 'b':
          setval ("background", "on");
          break;


@@ -486,6 +517,9 @@ GNU General Public License for more details.\n"));
        case 'x':
          setval ("dirstruct", "on");
          break;
        case 'x':
          setval ("dirstruct", "on");
          break;

+       case 174:
+         setval ("retryconnrefused", "on");
+         break;

 
          /* Options accepting an argument: */
        case 129:
 
          /* Options accepting an argument: */
        case 129:


@@ -498,13 +532,7 @@ GNU General Public License for more details.\n"));
          setval ("header", optarg);
          break;
        case 134:
          setval ("header", optarg);
          break;
        case 134:

-         /* Supported for compatibility; --dot-style=foo equivalent
-            to --progress=dot:foo.  */
-         {
-           char *tmp = alloca (3 + 1 + strlen (optarg));
-           sprintf (tmp, "dot:%s", optarg);
-           setval ("progress", tmp);
-         }
+         setval ("dotstyle", optarg);

          break;
        case 135:
          setval ("htmlify", optarg);
          break;
        case 135:
          setval ("htmlify", optarg);


@@ -539,6 +567,9 @@ GNU General Public License for more details.\n"));
        case 163:
          setval ("progress", optarg);
          break;
        case 163:
          setval ("progress", optarg);
          break;

+       case 164:
+         setval ("limitrate", optarg);
+         break;

        case 157:
          setval ("referer", optarg);
          break;
        case 157:
          setval ("referer", optarg);
          break;


@@ -549,7 +580,31 @@ GNU General Public License for more details.\n"));
        case 159:
          setval ("sslcertkey", optarg);
          break;
        case 159:
          setval ("sslcertkey", optarg);
          break;

+       case 166:
+         setval ("egdfile", optarg);
+         break;
+       case 169:
+         setval ("sslcadir", optarg);
+         break;
+       case 170:
+         setval ("sslcafile", optarg);
+         break;
+       case 171:
+         setval ("sslcerttype", optarg);
+         break;
+       case 172:
+         setval ("sslcheckcert", optarg);
+         break;
+       case 173:
+         setval ("sslprotocol", optarg);
+         break;

 #endif /* HAVE_SSL */
 #endif /* HAVE_SSL */

+       case 167:
+         setval ("postdata", optarg);
+         break;
+       case 168:
+         setval ("postfile", optarg);
+         break;

        case 'A':
          setval ("accept", optarg);
          break;
        case 'A':
          setval ("accept", optarg);
          break;


@@ -680,11 +735,6 @@ GNU General Public License for more details.\n"));
        }
     }
 
        }
     }
 

-  /* Initialize progress.  Have to do this after the options are
-     processed so we know where the log file is.  */
-  if (opt.verbose)
-    set_progress_implementation (opt.progress_type);
-

   /* All user options have now been processed, so it's now safe to do
      interoption dependency checks. */
 
   /* All user options have now been processed, so it's now safe to do
      interoption dependency checks. */
 


@@ -732,6 +782,11 @@ Can't timestamp and not clobber old files at the same time.\n"));
   if (opt.background)
     fork_to_background ();
 
   if (opt.background)
     fork_to_background ();
 

+  /* Initialize progress.  Have to do this after the options are
+     processed so we know where the log file is.  */
+  if (opt.verbose)
+    set_progress_implementation (opt.progress_type);
+

   /* Allocate basic pointer.  */
   url = ALLOCA_ARRAY (char *, nurl + 1);
   /* Fill in the arguments.  */
   /* Allocate basic pointer.  */
   url = ALLOCA_ARRAY (char *, nurl + 1);
   /* Fill in the arguments.  */


@@ -739,10 +794,7 @@ Can't timestamp and not clobber old files at the same time.\n"));
     {
       char *rewritten = rewrite_shorthand_url (argv[optind]);
       if (rewritten)
     {
       char *rewritten = rewrite_shorthand_url (argv[optind]);
       if (rewritten)

-       {
-         printf ("Converted %s to %s\n", argv[optind], rewritten);
-         url[i] = rewritten;
-       }
+       url[i] = rewritten;

       else
        url[i] = xstrdup (argv[optind]);
     }
       else
        url[i] = xstrdup (argv[optind]);
     }


@@ -799,8 +851,13 @@ Can't timestamp and not clobber old files at the same time.\n"));
 #endif
 #endif /* HAVE_SIGNAL */
 
 #endif
 #endif /* HAVE_SIGNAL */
 

+#ifdef HAVE_SSL
+  /* Must call this before resolving any URLs because it has the power
+     to disable `https'.  */
+  ssl_init_prng ();
+#endif
+

   status = RETROK;             /* initialize it, just-in-case */
   status = RETROK;             /* initialize it, just-in-case */

-  /*recursive_reset ();*/

   /* Retrieve the URLs from argument list.  */
   for (t = url; *t; t++)
     {
   /* Retrieve the URLs from argument list.  */
   for (t = url; *t; t++)
     {


@@ -851,8 +908,8 @@ Can't timestamp and not clobber old files at the same time.\n"));
                   legible (opt.quota));
     }
 
                   legible (opt.quota));
     }
 

-  if (opt.cookies_output)
-    save_cookies (opt.cookies_output);
+  if (opt.cookies_output && wget_cookie_jar)
+    cookie_jar_save (wget_cookie_jar, opt.cookies_output);

 
   if (opt.convert_links && !opt.delete_after)
     {
 
   if (opt.convert_links && !opt.delete_after)
     {


@@ -876,19 +933,20 @@ Can't timestamp and not clobber old files at the same time.\n"));
 #ifdef HAVE_SIGNAL
 /* Hangup signal handler.  When wget receives SIGHUP or SIGUSR1, it
    will proceed operation as usual, trying to write into a log file.
 #ifdef HAVE_SIGNAL
 /* Hangup signal handler.  When wget receives SIGHUP or SIGUSR1, it
    will proceed operation as usual, trying to write into a log file.

-   If that is impossible, the output will be turned off.  */
+   If that is impossible, the output will be turned off.
+
+   #### It is unsafe to do call libc functions from a signal handler.
+   What we should do is, set a global variable, and have the code in
+   log.c pick it up.  */

 
 static RETSIGTYPE
 redirect_output_signal (int sig)
 {
 
 static RETSIGTYPE
 redirect_output_signal (int sig)
 {

-  char tmp[100];
+  char *signal_name = (sig == SIGHUP ? "SIGHUP" :
+                      (sig == SIGUSR1 ? "SIGUSR1" :
+                       "WTF?!"));
+  log_request_redirect_output (signal_name);
+  progress_schedule_redirect ();

   signal (sig, redirect_output_signal);
   signal (sig, redirect_output_signal);

-  /* Please note that the double `%' in `%%s' is intentional, because
-     redirect_output passes tmp through printf.  */
-  sprintf (tmp, _("%s received, redirecting output to `%%s'.\n"),
-          (sig == SIGHUP ? "SIGHUP" :
-           (sig == SIGUSR1 ? "SIGUSR1" :
-            "WTF?!")));
-  redirect_output (tmp);

 }
 #endif /* HAVE_SIGNAL */
 }
 #endif /* HAVE_SIGNAL */