/* Command line parsing.
Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
- 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
+ 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 Free Software Foundation,
Inc.
This file is part of GNU Wget.
#include "http.h" /* for save_cookies */
#include "ptimer.h"
#include "warc.h"
-
#include <getopt.h>
#include <getpass.h>
#include <quote.h>
# define IF_SSL(x) NULL
#endif
-#ifdef ENABLE_DEBUG
-# define WHEN_DEBUG(x) x
-#else
-# define WHEN_DEBUG(x) NULL
-#endif
-
struct cmdline_option {
const char *long_name;
char short_name;
static struct cmdline_option option_data[] =
{
{ "accept", 'A', OPT_VALUE, "accept", -1 },
+ { "accept-regex", 0, OPT_VALUE, "acceptregex", -1 },
{ "adjust-extension", 'E', OPT_BOOLEAN, "adjustextension", -1 },
{ "append-output", 'a', OPT__APPEND_OUTPUT, NULL, required_argument },
{ "ask-password", 0, OPT_BOOLEAN, "askpassword", -1 },
{ "backups", 0, OPT_BOOLEAN, "backups", -1 },
{ "base", 'B', OPT_VALUE, "base", -1 },
{ "bind-address", 0, OPT_VALUE, "bindaddress", -1 },
+ { "body-data", 0, OPT_VALUE, "bodydata", -1 },
+ { "body-file", 0, OPT_VALUE, "bodyfile", -1 },
{ IF_SSL ("ca-certificate"), 0, OPT_VALUE, "cacertificate", -1 },
{ IF_SSL ("ca-directory"), 0, OPT_VALUE, "cadirectory", -1 },
{ "cache", 0, OPT_BOOLEAN, "cache", -1 },
{ "content-on-error", 0, OPT_BOOLEAN, "contentonerror", -1 },
{ "cookies", 0, OPT_BOOLEAN, "cookies", -1 },
{ "cut-dirs", 0, OPT_VALUE, "cutdirs", -1 },
- { WHEN_DEBUG ("debug"), 'd', OPT_BOOLEAN, "debug", -1 },
+ { "debug", 'd', OPT_BOOLEAN, "debug", -1 },
{ "default-page", 0, OPT_VALUE, "defaultpage", -1 },
{ "delete-after", 0, OPT_BOOLEAN, "deleteafter", -1 },
{ "directories", 0, OPT_BOOLEAN, "dirstruct", -1 },
{ "http-passwd", 0, OPT_VALUE, "httppassword", -1 }, /* deprecated */
{ "http-password", 0, OPT_VALUE, "httppassword", -1 },
{ "http-user", 0, OPT_VALUE, "httpuser", -1 },
+ { IF_SSL ("https-only"), 0, OPT_BOOLEAN, "httpsonly", -1 },
{ "ignore-case", 0, OPT_BOOLEAN, "ignorecase", -1 },
{ "ignore-length", 0, OPT_BOOLEAN, "ignorelength", -1 },
{ "ignore-tags", 0, OPT_VALUE, "ignoretags", -1 },
{ "load-cookies", 0, OPT_VALUE, "loadcookies", -1 },
{ "local-encoding", 0, OPT_VALUE, "localencoding", -1 },
{ "max-redirect", 0, OPT_VALUE, "maxredirect", -1 },
+ { "method", 0, OPT_VALUE, "method", -1 },
{ "mirror", 'm', OPT_BOOLEAN, "mirror", -1 },
{ "no", 'n', OPT__NO, NULL, required_argument },
{ "no-clobber", 0, OPT_BOOLEAN, "noclobber", -1 },
+ { "no-config", 0, OPT_BOOLEAN, "noconfig", -1},
{ "no-parent", 0, OPT_BOOLEAN, "noparent", -1 },
{ "output-document", 'O', OPT_VALUE, "outputdocument", -1 },
{ "output-file", 'o', OPT_VALUE, "logfile", -1 },
{ IF_SSL ("private-key"), 0, OPT_VALUE, "privatekey", -1 },
{ IF_SSL ("private-key-type"), 0, OPT_VALUE, "privatekeytype", -1 },
{ "progress", 0, OPT_VALUE, "progress", -1 },
+ { "show-progress", 0, OPT_BOOLEAN, "showprogress", -1 },
{ "protocol-directories", 0, OPT_BOOLEAN, "protocoldirectories", -1 },
{ "proxy", 0, OPT_BOOLEAN, "useproxy", -1 },
{ "proxy__compat", 'Y', OPT_VALUE, "useproxy", -1 }, /* back-compatible */
{ "read-timeout", 0, OPT_VALUE, "readtimeout", -1 },
{ "recursive", 'r', OPT_BOOLEAN, "recursive", -1 },
{ "referer", 0, OPT_VALUE, "referer", -1 },
+ { "regex-type", 0, OPT_VALUE, "regextype", -1 },
{ "reject", 'R', OPT_VALUE, "reject", -1 },
+ { "reject-regex", 0, OPT_VALUE, "rejectregex", -1 },
{ "relative", 'L', OPT_BOOLEAN, "relativeonly", -1 },
{ "remote-encoding", 0, OPT_VALUE, "remoteencoding", -1 },
{ "remove-listing", 0, OPT_BOOLEAN, "removelisting", -1 },
+ { "report-speed", 0, OPT_BOOLEAN, "reportspeed", -1 },
{ "restrict-file-names", 0, OPT_BOOLEAN, "restrictfilenames", -1 },
{ "retr-symlinks", 0, OPT_BOOLEAN, "retrsymlinks", -1 },
{ "retry-connrefused", 0, OPT_BOOLEAN, "retryconnrefused", -1 },
{ "server-response", 'S', OPT_BOOLEAN, "serverresponse", -1 },
{ "span-hosts", 'H', OPT_BOOLEAN, "spanhosts", -1 },
{ "spider", 0, OPT_BOOLEAN, "spider", -1 },
+ { "start-pos", 0, OPT_VALUE, "startpos", -1 },
{ "strict-comments", 0, OPT_BOOLEAN, "strictcomments", -1 },
{ "timeout", 'T', OPT_VALUE, "timeout", -1 },
{ "timestamping", 'N', OPT_BOOLEAN, "timestamping", -1 },
#endif
};
-#undef WHEN_DEBUG
#undef IF_SSL
/* Return a string that contains S with "no-" prepended. The string
static char *
no_prefix (const char *s)
{
- static char buffer[1024];
+ static char buffer[2048];
static char *p = buffer;
char *cp = p;
size_t i, o = 0;
for (i = 0; i < countof (option_data); i++)
{
- struct cmdline_option *opt = &option_data[i];
+ struct cmdline_option *cmdopt = &option_data[i];
struct option *longopt;
- if (!opt->long_name)
+ if (!cmdopt->long_name)
/* The option is disabled. */
continue;
longopt = &long_options[o++];
- longopt->name = opt->long_name;
+ longopt->name = cmdopt->long_name;
longopt->val = i;
- if (opt->short_name)
+ if (cmdopt->short_name)
{
- *p++ = opt->short_name;
- optmap[opt->short_name - 32] = longopt - long_options;
+ *p++ = cmdopt->short_name;
+ optmap[cmdopt->short_name - 32] = longopt - long_options;
}
- switch (opt->type)
+ switch (cmdopt->type)
{
case OPT_VALUE:
longopt->has_arg = required_argument;
- if (opt->short_name)
+ if (cmdopt->short_name)
*p++ = ':';
break;
case OPT_BOOLEAN:
identical to "--foo", except it has opposite meaning and
it doesn't allow an argument. */
longopt = &long_options[o++];
- longopt->name = no_prefix (opt->long_name);
+ longopt->name = no_prefix (cmdopt->long_name);
longopt->has_arg = no_argument;
/* Mask the value so we'll be able to recognize that we're
dealing with the false value. */
longopt->val = i | BOOLEAN_NEG_MARKER;
break;
default:
- assert (opt->argtype != -1);
- longopt->has_arg = opt->argtype;
- if (opt->short_name)
+ assert (cmdopt->argtype != -1);
+ longopt->has_arg = cmdopt->argtype;
+ if (cmdopt->short_name)
{
if (longopt->has_arg == required_argument)
*p++ = ':';
-v, --verbose be verbose (this is the default).\n"),
N_("\
-nv, --no-verbose turn off verboseness, without being quiet.\n"),
+ N_("\
+ --report-speed=TYPE Output bandwidth as TYPE. TYPE can be bits.\n"),
N_("\
-i, --input-file=FILE download URLs found in local or external FILE.\n"),
N_("\
-B, --base=URL resolves HTML input-file links (-i -F)\n\
relative to URL.\n"),
N_("\
- --config=FILE Specify config file to use.\n"),
+ --config=FILE Specify config file to use.\n"),
+ N_("\
+ --no-config Do not read any config file.\n"),
"\n",
N_("\
existing files (overwriting them).\n"),
N_("\
-c, --continue resume getting a partially-downloaded file.\n"),
+ N_("\
+ --start-pos=OFFSET start downloading from zero-based position OFFSET.\n"),
N_("\
--progress=TYPE select progress gauge type.\n"),
+ N_("\
+ --show-progress display the progress bar in any verbosity mode.\n"),
N_("\
-N, --timestamping don't re-retrieve files unless newer than\n\
local.\n"),
--post-data=STRING use the POST method; send STRING as the data.\n"),
N_("\
--post-file=FILE use the POST method; send contents of FILE.\n"),
+ N_("\
+ --method=HTTPMethod use method \"HTTPMethod\" in the header.\n"),
+ N_("\
+ --body-data=STRING Send STRING as data. --method MUST be set.\n"),
+ N_("\
+ --body-file=FILE Send contents of FILE. --method MUST be set.\n"),
N_("\
--content-disposition honor the Content-Disposition header when\n\
choosing local file names (EXPERIMENTAL).\n"),
HTTPS (SSL/TLS) options:\n"),
N_("\
--secure-protocol=PR choose secure protocol, one of auto, SSLv2,\n\
- SSLv3, and TLSv1.\n"),
+ SSLv3, TLSv1 and PFS.\n"),
+ N_("\
+ --https-only only follow secure HTTPS links\n"),
N_("\
--no-check-certificate don't validate the server's certificate.\n"),
N_("\
N_("\
-k, --convert-links make links in downloaded HTML or CSS point to\n\
local files.\n"),
+ N_("\
+ --backups=N before writing file X, rotate up to N backup files.\n"),
+
#ifdef __VMS
N_("\
-K, --backup-converted before converting file X, back up as X_orig.\n"),
-A, --accept=LIST comma-separated list of accepted extensions.\n"),
N_("\
-R, --reject=LIST comma-separated list of rejected extensions.\n"),
+ N_("\
+ --accept-regex=REGEX regex matching accepted URLs.\n"),
+ N_("\
+ --reject-regex=REGEX regex matching rejected URLs.\n"),
+#ifdef HAVE_LIBPCRE
+ N_("\
+ --regex-type=TYPE regex type (posix|pcre).\n"),
+#else
+ N_("\
+ --regex-type=TYPE regex type (posix).\n"),
+#endif
N_("\
-D, --domains=LIST comma-separated list of accepted domains.\n"),
N_("\
N_("\
-np, --no-parent don't ascend to the parent directory.\n"),
"\n",
-
N_("Mail bug reports and suggestions to <bug-wget@gnu.org>.\n")
};
assert (prefix != NULL);
assert (line != NULL);
+ assert (line_length > TABULATION);
line_dup = xstrdup (line);
- if (line_length <= 0)
- line_length = MAX_CHARS_PER_LINE - TABULATION;
-
if (printf ("%s", prefix) < 0)
return -1;
- remaining_chars = line_length;
+
+ /* Wrap to new line after prefix. */
+ remaining_chars = 0;
+
/* We break on spaces. */
token = strtok (line_dup, " ");
while (token != NULL)
/* If however a token is much larger than the maximum
line length, all bets are off and we simply print the
token on the next line. */
- if (remaining_chars <= strlen (token))
+ if (remaining_chars <= (int) strlen (token))
{
if (printf ("\n%*c", TABULATION, ' ') < 0)
return -1;
#ifdef ENABLE_NLS
if (format_and_print_line (locale_title,
- LOCALEDIR,
+ LOCALEDIR,
MAX_CHARS_PER_LINE) < 0)
exit (3);
#endif /* def ENABLE_NLS */
/* TRANSLATORS: When available, an actual copyright character
(circle-c) should be used in preference to "(C)". */
- if (fputs (_("\
-Copyright (C) 2011 Free Software Foundation, Inc.\n"), stdout) < 0)
+ if (printf (_("\
+Copyright (C) %s Free Software Foundation, Inc.\n"), "2014") < 0)
exit (3);
if (fputs (_("\
License GPLv3+: GNU GPL version 3 or later\n\
exit (0);
}
-char *program_name; /* Needed by lib/error.c. */
+static char *program_name; /* Needed by lib/error.c. */
char *program_argstring; /* Needed by wget_warc.c. */
int
longindex = -1;
int retconf;
bool use_userconfig = false;
+ bool noconfig = false;
while ((retconf = getopt_long (argc, argv,
short_options, long_options, &longindex)) != -1)
{
int confval;
- bool userrc_ret = true;
struct cmdline_option *config_opt;
/* There is no short option for "--config". */
{
confval = long_options[longindex].val;
config_opt = &option_data[confval & ~BOOLEAN_NEG_MARKER];
- if (strcmp (config_opt->long_name, "config") == 0)
+ if (strcmp (config_opt->long_name, "no-config") == 0)
{
- userrc_ret &= run_wgetrc (optarg);
- use_userconfig = true;
+ noconfig = true;
+ break;
}
- if (!userrc_ret)
+ else if (strcmp (config_opt->long_name, "config") == 0)
{
- printf ("Exiting due to error in %s\n", optarg);
- exit (2);
+ bool userrc_ret = true;
+ userrc_ret &= run_wgetrc (optarg);
+ use_userconfig = true;
+ if (userrc_ret)
+ break;
+ else
+ {
+ fprintf (stderr, _("Exiting due to error in %s\n"), optarg);
+ exit (2);
+ }
}
- else
- break;
}
}
/* If the user did not specify a config, read the system wgetrc and ~/.wgetrc. */
- if (use_userconfig == false)
+ if (noconfig == false && use_userconfig == false)
initialize ();
opterr = 0;
short_options, long_options, &longindex)) != -1)
{
int val;
- struct cmdline_option *opt;
+ struct cmdline_option *cmdopt;
/* If LONGINDEX is unchanged, it means RET is referring a short
option. */
{
if (ret == '?')
{
- print_usage (0);
- printf ("\n");
- printf (_("Try `%s --help' for more options.\n"), exec_name);
+ print_usage (1);
+ fprintf (stderr, "\n");
+ fprintf (stderr, _("Try `%s --help' for more options.\n"),
+ exec_name);
exit (2);
}
/* Find the short option character in the mapping. */
/* Use the retrieved value to locate the option in the
option_data array, and to see if we're dealing with the
negated "--no-FOO" variant of the boolean option "--foo". */
- opt = &option_data[val & ~BOOLEAN_NEG_MARKER];
- switch (opt->type)
+ cmdopt = &option_data[val & ~BOOLEAN_NEG_MARKER];
+ switch (cmdopt->type)
{
case OPT_VALUE:
- setoptval (opt->data, optarg, opt->long_name);
+ setoptval (cmdopt->data, optarg, cmdopt->long_name);
break;
case OPT_BOOLEAN:
if (optarg)
/* The user has specified a value -- use it. */
- setoptval (opt->data, optarg, opt->long_name);
+ setoptval (cmdopt->data, optarg, cmdopt->long_name);
else
{
/* NEG is true for `--no-FOO' style boolean options. */
bool neg = !!(val & BOOLEAN_NEG_MARKER);
- setoptval (opt->data, neg ? "0" : "1", opt->long_name);
+ setoptval (cmdopt->data, neg ? "0" : "1", cmdopt->long_name);
}
break;
case OPT_FUNCALL:
{
- void (*func) (void) = (void (*) (void)) opt->data;
+ void (*func) (void) = (void (*) (void)) cmdopt->data;
func ();
}
break;
case OPT__APPEND_OUTPUT:
- setoptval ("logfile", optarg, opt->long_name);
+ setoptval ("logfile", optarg, cmdopt->long_name);
append_to_log = true;
break;
case OPT__EXECUTE:
/* We support real --no-FOO flags now, but keep these
short options for convenience and backward
compatibility. */
- char *p;
for (p = optarg; p && *p; p++)
switch (*p)
{
case 'v':
- setoptval ("verbose", "0", opt->long_name);
+ setoptval ("verbose", "0", cmdopt->long_name);
break;
case 'H':
- setoptval ("addhostdir", "0", opt->long_name);
+ setoptval ("addhostdir", "0", cmdopt->long_name);
break;
case 'd':
- setoptval ("dirstruct", "0", opt->long_name);
+ setoptval ("dirstruct", "0", cmdopt->long_name);
break;
case 'c':
- setoptval ("noclobber", "1", opt->long_name);
+ setoptval ("noclobber", "1", cmdopt->long_name);
break;
case 'p':
- setoptval ("noparent", "1", opt->long_name);
+ setoptval ("noparent", "1", cmdopt->long_name);
break;
default:
fprintf (stderr, _("%s: illegal option -- `-n%c'\n"),
flag = (*optarg == '1' || c_tolower (*optarg) == 'y'
|| (c_tolower (optarg[0]) == 'o'
&& c_tolower (optarg[1]) == 'n'));
- setoptval (opt->type == OPT__PARENT ? "noparent" : "noclobber",
- flag ? "0" : "1", opt->long_name);
+ setoptval (cmdopt->type == OPT__PARENT ? "noparent" : "noclobber",
+ flag ? "0" : "1", cmdopt->long_name);
break;
}
case OPT__DONT_REMOVE_LISTING:
- setoptval ("removelisting", "0", opt->long_name);
+ setoptval ("removelisting", "0", cmdopt->long_name);
break;
}
nurl = argc - optind;
+ /* If we do not have Debug support compiled in AND Wget is invoked with the
+ * --debug switch, instead of failing, we silently turn it into a no-op. For
+ * this no-op, we explicitly set opt.debug to false and hence none of the
+ * Debug output messages will be printed.
+ */
+#ifndef ENABLE_DEBUG
+ if (opt.debug)
+ {
+ fprintf (stderr, _("Debugging support not compiled in. "
+ "Ignoring --debug flag.\n"));
+ opt.debug = false;
+ }
+#endif
+
/* All user options have now been processed, so it's now safe to do
interoption dependency checks. */
{
fprintf (stderr,
_("Both --no-clobber and --convert-links were specified,"
- "only --convert-links will be used.\n"));
+ " only --convert-links will be used.\n"));
opt.noclobber = false;
}
if (opt.verbose == -1)
opt.verbose = !opt.quiet;
+ if (opt.verbose == 1)
+ opt.show_progress = true;
+
/* Sanity checks. */
if (opt.verbose && opt.quiet)
{
_("WARC output does not work with --spider.\n"));
exit (1);
}
- if (opt.always_rest)
+ if (opt.always_rest || opt.start_pos >= 0)
{
fprintf (stderr,
- _("WARC output does not work with --continue, "
- "--continue will be disabled.\n"));
+ _("WARC output does not work with --continue or"
+ " --start-pos, they will be disabled.\n"));
opt.always_rest = false;
+ opt.start_pos = -1;
}
if (opt.warc_cdx_dedup_filename != 0 && !opt.warc_digests_enabled)
{
}
if (opt.warc_keep_log)
{
- opt.progress_type = "dot";
+ opt.progress_type = xstrdup ("dot");
}
}
exit (1);
}
+ if (opt.start_pos >= 0 && opt.always_rest)
+ {
+ fprintf (stderr,
+ _("Specifying both --start-pos and --continue is not "
+ "recommended; --continue will be disabled.\n"));
+ opt.always_rest = false;
+ }
+
if (!nurl && !opt.input_filename)
{
/* No URL specified. */
fprintf (stderr, _("%s: missing URL\n"), exec_name);
print_usage (1);
- printf ("\n");
+ fprintf (stderr, "\n");
/* #### Something nicer should be printed here -- similar to the
pre-1.5 `--help' page. */
fprintf (stderr, _("Try `%s --help' for more options.\n"), exec_name);
exit (1);
}
+ /* Compile the regular expressions. */
+ switch (opt.regex_type)
+ {
+#ifdef HAVE_LIBPCRE
+ case regex_type_pcre:
+ opt.regex_compile_fun = compile_pcre_regex;
+ opt.regex_match_fun = match_pcre_regex;
+ break;
+#endif
+
+ case regex_type_posix:
+ default:
+ opt.regex_compile_fun = compile_posix_regex;
+ opt.regex_match_fun = match_posix_regex;
+ break;
+ }
+ if (opt.acceptregex_s)
+ {
+ opt.acceptregex = opt.regex_compile_fun (opt.acceptregex_s);
+ if (!opt.acceptregex)
+ exit (1);
+ }
+ if (opt.rejectregex_s)
+ {
+ opt.rejectregex = opt.regex_compile_fun (opt.rejectregex_s);
+ if (!opt.rejectregex)
+ exit (1);
+ }
+ if (opt.post_data || opt.post_file_name)
+ {
+ if (opt.post_data && opt.post_file_name)
+ {
+ fprintf (stderr, _("You cannot specify both --post-data and --post-file.\n"));
+ exit (1);
+ }
+ else if (opt.method)
+ {
+ fprintf (stderr, _("You cannot use --post-data or --post-file along with --method. "
+ "--method expects data through --body-data and --body-file options"));
+ exit (1);
+ }
+ }
+ if (opt.body_data || opt.body_file)
+ {
+ if (!opt.method)
+ {
+ fprintf (stderr, _("You must specify a method through --method=HTTPMethod "
+ "to use with --body-data or --body-file.\n"));
+ exit (1);
+ }
+ else if (opt.body_data && opt.body_file)
+ {
+ fprintf (stderr, _("You cannot specify both --body-data and --body-file.\n"));
+ exit (1);
+ }
+ }
+
+ /* Set various options as required for opt.method. */
+
+ /* When user specifies HEAD as the method, we do not wish to download any
+ files. Hence, set wget to run in spider mode. */
+ if (opt.method && strcasecmp (opt.method, "HEAD") == 0)
+ setoptval ("spider", "1", "spider");
+
+ /* Convert post_data to body-data and post_file_name to body-file options.
+ This is required so as to remove redundant code later on in gethttp().
+ The --post-data and --post-file options may also be removed in
+ the future hence it makes sense to convert them to aliases for
+ the more generic --method options.
+ This MUST occur only after the sanity checks so as to prevent the
+ user from setting both post and body options simultaneously.
+ */
+ if (opt.post_data || opt.post_file_name)
+ {
+ setoptval ("method", "POST", "method");
+ if (opt.post_data)
+ {
+ setoptval ("bodydata", opt.post_data, "body-data");
+ opt.post_data = NULL;
+ }
+ else
+ {
+ setoptval ("bodyfile", opt.post_file_name, "body-file");
+ opt.post_file_name = NULL;
+ }
+ }
+
#ifdef ENABLE_IRI
if (opt.enable_iri)
{
/* Initialize progress. Have to do this after the options are
processed so we know where the log file is. */
- if (opt.verbose)
+ if (opt.show_progress)
set_progress_implementation (opt.progress_type);
/* Fill in the arguments. */
&dt, opt.recursive, iri, true);
}
- if (opt.delete_after && file_exists_p(filename))
+ if (opt.delete_after && filename != NULL && file_exists_p (filename))
{
DEBUGP (("Removing file due to --delete-after in main():\n"));
logprintf (LOG_VERBOSE, _("Removing %s.\n"), filename);
if (opt.convert_links && !opt.delete_after)
convert_all_links ();
- /* Close WARC file. */
- if (opt.warc_filename != 0)
- warc_close ();
-
- log_close ();
-
- for (i = 0; i < nurl; i++)
- xfree (url[i]);
cleanup ();
exit (get_exit_status ());
projects / wget / blobdiff