/* Command line parsing.
- Copyright (C) 1995, 1996, 1997, 1998 Free Software Foundation, Inc.
+ Copyright (C) 1995, 1996, 1997, 1998, 2000 Free Software Foundation, Inc.
This file is part of Wget.
# include <locale.h>
#endif /* HAVE_LOCALE_H */
#endif /* HAVE_NLS */
+#include <errno.h>
#define OPTIONS_DEFINED_HERE /* for options.h */
things up. For example, when in a foreign locale, Solaris
strptime() fails to handle international dates correctly, which
makes http_atotm() malfunction. */
+#ifdef LC_MESSAGES
setlocale (LC_MESSAGES, "");
+#else
+ setlocale (LC_ALL, "");
+#endif
/* Set the text message domain. */
bindtextdomain ("wget", LOCALEDIR);
textdomain ("wget");
#endif /* HAVE_NLS */
}
+
+/* It's kosher to declare these here because their interface _has_ to
+ be void foo(void). */
+void url_init PARAMS ((void));
+void host_init PARAMS ((void));
+
+/* This just calls the various initialization functions from the
+ modules that need one-time initialization. */
+static void
+private_initialize (void)
+{
+ url_init ();
+ host_init ();
+}
\f
/* Print the usage message. */
static void
-V, --version display the version of Wget and exit.\n\
-h, --help print this help.\n\
-b, --background go to background after startup.\n\
- -e, --execute=COMMAND execute a `.wgetrc\' command.\n\
+ -e, --execute=COMMAND execute a `.wgetrc\'-style command.\n\
\n"), _("\
Logging and input file:\n\
-o, --output-file=FILE log messages to FILE.\n\
-q, --quiet quiet (no output).\n\
-v, --verbose be verbose (this is the default).\n\
-nv, --non-verbose turn off verboseness, without being quiet.\n\
- -i, --input-file=FILE read URL-s from file.\n\
+ -i, --input-file=FILE download URLs found in FILE.\n\
-F, --force-html treat input file as HTML.\n\
+ -B, --base=URL prepends URL to relative links in -F -i file.\n\
+ --sslcertfile=FILE optional client certificate.\n\
+ --sslcertkey=KEYFILE optional keyfile for this certificate.\n\
\n"), _("\
Download:\n\
+ --bind-address=ADDRESS bind to ADDRESS (hostname or IP) on local host.\n\
-t, --tries=NUMBER set number of retries to NUMBER (0 unlimits).\n\
-O --output-document=FILE write documents to FILE.\n\
- -nc, --no-clobber don\'t clobber existing files.\n\
- -c, --continue restart getting an existing file.\n\
+ -nc, --no-clobber don\'t clobber existing files or use .# suffixes.\n\
+ -c, --continue resume getting a partially-downloaded file.\n\
--dot-style=STYLE set retrieval display style.\n\
-N, --timestamping don\'t retrieve files if older than local.\n\
-S, --server-response print server response.\n\
--spider don\'t download anything.\n\
-T, --timeout=SECONDS set the read timeout to SECONDS.\n\
-w, --wait=SECONDS wait SECONDS between retrievals.\n\
- --waitretry=SECONDS wait 0..max SECONDS between retries of a retrieval.\n\
+ --waitretry=SECONDS wait 1...SECONDS between retries of a retrieval.\n\
-Y, --proxy=on/off turn proxy on or off.\n\
-Q, --quota=NUMBER set retrieval quota to NUMBER.\n\
\n"), _("\
--http-user=USER set http user to USER.\n\
--http-passwd=PASS set http password to PASS.\n\
-C, --cache=on/off (dis)allow server-cached data (normally allowed).\n\
+ -E, --html-extension save all text/html documents with .html extension.\n\
--ignore-length ignore `Content-Length\' header field.\n\
--header=STRING insert STRING among the headers.\n\
--proxy-user=USER set USER as proxy username.\n\
--proxy-passwd=PASS set PASS as proxy password.\n\
+ --referer=URL include `Referer: URL\' header in HTTP request.\n\
-s, --save-headers save the HTTP headers to file.\n\
-U, --user-agent=AGENT identify as AGENT instead of Wget/VERSION.\n\
\n"), _("\
FTP options:\n\
- --retr-symlinks retrieve FTP symbolic links.\n\
+ --retr-symlinks when recursing, retrieve linked-to files (not dirs).\n\
-g, --glob=on/off turn file name globbing on or off.\n\
--passive-ftp use the \"passive\" transfer mode.\n\
\n"), _("\
Recursive retrieval:\n\
-r, --recursive recursive web-suck -- use with care!.\n\
- -l, --level=NUMBER maximum recursion depth (0 to unlimit).\n\
- --delete-after delete downloaded files.\n\
+ -l, --level=NUMBER maximum recursion depth (inf or 0 for infinite).\n\
+ --delete-after delete files locally after downloading them.\n\
-k, --convert-links convert non-relative links to relative.\n\
-K, --backup-converted before converting file X, back up as X.orig.\n\
- -m, --mirror turn on options suitable for mirroring.\n\
+ -m, --mirror shortcut option equivalent to -r -N -l inf -nr.\n\
-nr, --dont-remove-listing don\'t remove `.listing\' files.\n\
+ -p, --page-requisites get all images, etc. needed to display HTML page.\n\
\n"), _("\
Recursive accept/reject:\n\
- -A, --accept=LIST list of accepted extensions.\n\
- -R, --reject=LIST list of rejected extensions.\n\
- -D, --domains=LIST list of accepted domains.\n\
+ -A, --accept=LIST comma-separated list of accepted extensions.\n\
+ -R, --reject=LIST comma-separated list of rejected extensions.\n\
+ -D, --domains=LIST comma-separated list of accepted domains.\n\
--exclude-domains=LIST comma-separated list of rejected domains.\n\
- -L, --relative follow relative links only.\n\
--follow-ftp follow FTP links from HTML documents.\n\
+ --follow-tags=LIST comma-separated list of followed HTML tags.\n\
+ -G, --ignore-tags=LIST comma-separated list of ignored HTML tags.\n\
-H, --span-hosts go to foreign hosts when recursive.\n\
+ -L, --relative follow relative links only.\n\
-I, --include-directories=LIST list of allowed directories.\n\
-X, --exclude-directories=LIST list of excluded directories.\n\
-nh, --no-host-lookup don\'t DNS-lookup hosts.\n\
{
char **url, **t;
int i, c, nurl, status, append_to_log;
- int wr;
+ int wr = 0;
static struct option long_options[] =
{
+ /* Options without arguments: */
{ "background", no_argument, NULL, 'b' },
{ "continue", no_argument, NULL, 'c' },
{ "convert-links", no_argument, NULL, 'k' },
{ "backup-converted", no_argument, NULL, 'K' },
{ "debug", no_argument, NULL, 'd' },
{ "dont-remove-listing", no_argument, NULL, 21 },
- { "email-address", no_argument, NULL, 'E' }, /* undocumented (debug) */
+ { "email-address", no_argument, NULL, 26 }, /* undocumented (debug) */
{ "follow-ftp", no_argument, NULL, 14 },
{ "force-directories", no_argument, NULL, 'x' },
{ "force-hier", no_argument, NULL, 'x' }, /* obsolete */
{ "force-html", no_argument, NULL, 'F'},
{ "help", no_argument, NULL, 'h' },
+ { "html-extension", no_argument, NULL, 'E' },
{ "ignore-length", no_argument, NULL, 10 },
{ "mirror", no_argument, NULL, 'm' },
{ "no-clobber", no_argument, NULL, 13 },
{ "no-directories", no_argument, NULL, 19 },
{ "no-host-directories", no_argument, NULL, 20 },
{ "no-host-lookup", no_argument, NULL, 22 },
+ { "no-http-keep-alive", no_argument, NULL, 28 },
{ "no-parent", no_argument, NULL, 5 },
{ "non-verbose", no_argument, NULL, 18 },
{ "passive-ftp", no_argument, NULL, 11 },
+ { "page-requisites", no_argument, NULL, 'p' },
{ "quiet", no_argument, NULL, 'q' },
{ "recursive", no_argument, NULL, 'r' },
{ "relative", no_argument, NULL, 'L' },
{ "verbose", no_argument, NULL, 'v' },
{ "version", no_argument, NULL, 'V' },
+ /* Options accepting an argument: */
{ "accept", required_argument, NULL, 'A' },
{ "append-output", required_argument, NULL, 'a' },
{ "backups", required_argument, NULL, 23 }, /* undocumented */
{ "base", required_argument, NULL, 'B' },
+ { "bind-address", required_argument, NULL, 27 },
{ "cache", required_argument, NULL, 'C' },
{ "cut-dirs", required_argument, NULL, 17 },
{ "delete-after", no_argument, NULL, 8 },
{ "execute", required_argument, NULL, 'e' },
{ "exclude-directories", required_argument, NULL, 'X' },
{ "exclude-domains", required_argument, NULL, 12 },
+ { "follow-tags", required_argument, NULL, 25 },
{ "glob", required_argument, NULL, 'g' },
{ "header", required_argument, NULL, 3 },
{ "htmlify", required_argument, NULL, 7 },
{ "http-passwd", required_argument, NULL, 2 },
{ "http-user", required_argument, NULL, 1 },
+ { "ignore-tags", required_argument, NULL, 'G' },
{ "include-directories", required_argument, NULL, 'I' },
{ "input-file", required_argument, NULL, 'i' },
{ "level", required_argument, NULL, 'l' },
{ "user-agent", required_argument, NULL, 'U' },
{ "referer", required_argument, NULL, 129 },
{ "use-proxy", required_argument, NULL, 'Y' },
+#ifdef HAVE_SSL
+ { "sslcertfile", required_argument, NULL, 132},
+ { "sslcertkey", required_argument, NULL, 133},
+#endif /* HAVE_SSL */
{ "wait", required_argument, NULL, 'w' },
{ "waitretry", required_argument, NULL, 24 },
{ 0, 0, 0, 0 }
};
i18n_initialize ();
+ private_initialize ();
append_to_log = 0;
windows_main_junk (&argc, (char **) argv, (char **) &exec_name);
#endif
- initialize ();
+ initialize (); /* sets option defaults; reads the system wgetrc and .wgetrc */
+ /* [Is the order of the option letters significant? If not, they should be
+ alphabetized, like the long_options. The only thing I know for sure is
+ that the options with required arguments must be followed by a ':'.
+ -- Dan Harkless <wget@harkless.org>] */
while ((c = getopt_long (argc, argv, "\
-hVqvdkKsxmNWrHSLcFbEY:g:T:U:O:l:n:i:o:a:t:D:A:R:P:B:e:Q:X:I:w:",
+hpVqvdkKsxmNWrHSLcFbEY:G:g:T:U:O:l:n:i:o:a:t:D:A:R:P:B:e:Q:X:I:w:",
long_options, (int *)0)) != EOF)
{
switch (c)
case 22:
setval ("simplehostcheck", "on");
break;
+ case 26:
+ /* For debugging purposes. */
+ printf ("%s\n", ftp_getaddress ());
+ exit (0);
+ break;
+ case 27:
+ setval ("bindaddress", optarg);
+ break;
+ case 28:
+ setval ("httpkeepalive", "off");
+ break;
case 'b':
setval ("background", "on");
break;
#endif /* not DEBUG */
break;
case 'E':
- /* For debugging purposes. */
- printf ("%s\n", ftp_getaddress ());
- exit (0);
+ setval ("htmlextension", "on");
break;
case 'F':
setval ("forcehtml", "on");
#endif
exit (0);
break;
- case 'k':
- setval ("convertlinks", "on");
- break;
case 'K':
setval ("backupconverted", "on");
break;
+ case 'k':
+ setval ("convertlinks", "on");
+ break;
case 'L':
setval ("relativeonly", "on");
break;
case 'N':
setval ("timestamping", "on");
break;
+ case 'p':
+ setval ("pagerequisites", "on");
+ break;
case 'S':
setval ("serverresponse", "on");
break;
case 'V':
printf ("GNU Wget %s\n\n", version_string);
printf ("%s", _("\
-Copyright (C) 1995, 1996, 1997, 1998 Free Software Foundation, Inc.\n\
+Copyright (C) 1995, 1996, 1997, 1998, 2000 Free Software Foundation, Inc.\n\
This program is distributed in the hope that it will be useful,\n\
but WITHOUT ANY WARRANTY; without even the implied warranty of\n\
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n\
GNU General Public License for more details.\n"));
- printf (_("\nWritten by Hrvoje Niksic <hniksic@srce.hr>.\n"));
+ printf (_("\nOriginally written by Hrvoje Niksic <hniksic@arsdigita.com>.\n"));
exit (0);
break;
case 'v':
case 23:
setval ("backups", optarg);
break;
+ case 24:
+ setval ("waitretry", optarg);
+ wr = 1;
+ break;
+ case 25:
+ setval ("followtags", optarg);
+ break;
+ case 129:
+ setval ("referer", optarg);
+ break;
+#ifdef HAVE_SSL
+ case 132:
+ setval ("sslcertfile", optarg);
+ break;
+ case 133:
+ setval ("sslcertkey", optarg);
+ break;
+#endif /* HAVE_SSL */
case 'A':
setval ("accept", optarg);
break;
optarg);
exit (1);
}
- free (com);
- free (val);
+ xfree (com);
+ xfree (val);
}
break;
+ case 'G':
+ setval ("ignoretags", optarg);
+ break;
case 'g':
setval ("glob", optarg);
break;
case 'p':
setval ("noparent", "on");
break;
+ case 'k':
+ setval ("httpkeepalive", "off");
+ break;
default:
printf (_("%s: illegal option -- `-n%c'\n"), exec_name, *p);
print_usage ();
case 'U':
setval ("useragent", optarg);
break;
- case 129:
- setval ("referer", optarg);
- break;
case 'w':
setval ("wait", optarg);
break;
- case 24:
- setval ("waitretry", optarg);
- wr = 1;
- break;
case 'X':
setval ("excludedirectories", optarg);
break;
break;
}
}
+
+ /* All user options have now been processed, so it's now safe to do
+ interoption dependency checks. */
+
+ if (opt.reclevel == 0)
+ opt.reclevel = INFINITE_RECURSION; /* see wget.h for commentary on this */
+
+ if (opt.page_requisites && !opt.recursive)
+ {
+ opt.recursive = TRUE;
+ opt.reclevel = 0;
+ if (!opt.no_dirstruct)
+ opt.dirstruct = TRUE; /* usually handled by cmd_spec_recursive() */
+ }
+
if (opt.verbose == -1)
opt.verbose = !opt.quiet;
A simple check on the values is not enough, I could have set
wait to n>0 and waitretry to 0 - HEH */
if (opt.wait && !wr)
- setval ("waitretry", opt.wait);
+ {
+ char opt_wait_str[256]; /* bigger than needed buf to prevent overflow */
+
+ sprintf(opt_wait_str, "%ld", opt.wait);
+ setval ("waitretry", opt_wait_str);
+ }
/* Sanity checks. */
if (opt.verbose && opt.quiet)
opt.dfp = stdout;
else
{
+ struct stat st;
opt.dfp = fopen (opt.output_document, "wb");
if (opt.dfp == NULL)
{
perror (opt.output_document);
exit (1);
}
+ if (fstat (fileno (opt.dfp), &st) == 0 && S_ISREG (st.st_mode))
+ opt.od_known_regular = 1;
}
}
/* Retrieve the URLs from argument list. */
for (t = url; *t; t++)
{
- char *filename, *new_file;
+ char *filename, *redirected_URL;
int dt;
- status = retrieve_url (*t, &filename, &new_file, NULL, &dt);
+ status = retrieve_url (*t, &filename, &redirected_URL, NULL, &dt);
if (opt.recursive && status == RETROK && (dt & TEXTHTML))
- status = recursive_retrieve (filename, new_file ? new_file : *t);
- FREE_MAYBE (new_file);
+ status = recursive_retrieve (filename,
+ redirected_URL ? redirected_URL : *t);
+
+ if (opt.delete_after && file_exists_p(filename))
+ {
+ DEBUGP (("Removing file due to --delete-after in main():\n"));
+ logprintf (LOG_VERBOSE, _("Removing %s.\n"), filename);
+ if (unlink (filename))
+ logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
+ }
+
+ FREE_MAYBE (redirected_URL);
FREE_MAYBE (filename);
}
{
logprintf (LOG_NOTQUIET,
_("\nFINISHED --%s--\nDownloaded: %s bytes in %d files\n"),
- time_str (NULL), legible (opt.downloaded), opt.numurls);
+ time_str (NULL),
+ (opt.downloaded_overflow ?
+ "<overflow>" : legible_very_long (opt.downloaded)),
+ opt.numurls);
/* Print quota warning, if exceeded. */
- if (opt.quota && opt.downloaded > opt.quota)
+ if (downloaded_exceeds_quota ())
logprintf (LOG_NOTQUIET,
_("Download quota (%s bytes) EXCEEDED!\n"),
legible (opt.quota));
}
- if (opt.convert_links)
+ if (opt.convert_links && !opt.delete_after)
{
convert_all_links ();
}
log_close ();
cleanup ();
+#ifdef DEBUG_MALLOC
+ print_malloc_debug_stats ();
+#endif
if (status == RETROK)
return 0;
else
projects / wget / blobdiff