/* Command line parsing.
Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
- 2005, 2006, 2007, 2008, 2009, 2010 Free Software Foundation, Inc.
+ 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 Free Software Foundation,
+ Inc.
This file is part of GNU Wget.
#include <stdio.h>
#include <stdlib.h>
-#ifdef HAVE_UNISTD_H
-# include <unistd.h>
-#endif /* HAVE_UNISTD_H */
+#include <unistd.h>
#include <string.h>
#include <signal.h>
#ifdef ENABLE_NLS
#include "convert.h"
#include "spider.h"
#include "http.h" /* for save_cookies */
-
+#include "ptimer.h"
+#include "warc.h"
#include <getopt.h>
#include <getpass.h>
#include <quote.h>
+#ifdef WINDOWS
+# include <io.h>
+# include <fcntl.h>
+#endif
+
#ifdef __VMS
-#include "vms.h"
+# include "vms.h"
#endif /* __VMS */
#ifndef PATH_SEPARATOR
# define PATH_SEPARATOR '/'
#endif
+#ifndef ENABLE_IRI
+struct iri dummy_iri;
+#endif
+
struct options opt;
/* defined in version.c */
static struct cmdline_option option_data[] =
{
{ "accept", 'A', OPT_VALUE, "accept", -1 },
+ { "accept-regex", 0, OPT_VALUE, "acceptregex", -1 },
{ "adjust-extension", 'E', OPT_BOOLEAN, "adjustextension", -1 },
{ "append-output", 'a', OPT__APPEND_OUTPUT, NULL, required_argument },
{ "ask-password", 0, OPT_BOOLEAN, "askpassword", -1 },
{ "backups", 0, OPT_BOOLEAN, "backups", -1 },
{ "base", 'B', OPT_VALUE, "base", -1 },
{ "bind-address", 0, OPT_VALUE, "bindaddress", -1 },
+ { "body-data", 0, OPT_VALUE, "bodydata", -1 },
+ { "body-file", 0, OPT_VALUE, "bodyfile", -1 },
{ IF_SSL ("ca-certificate"), 0, OPT_VALUE, "cacertificate", -1 },
{ IF_SSL ("ca-directory"), 0, OPT_VALUE, "cadirectory", -1 },
{ "cache", 0, OPT_BOOLEAN, "cache", -1 },
{ IF_SSL ("certificate-type"), 0, OPT_VALUE, "certificatetype", -1 },
{ IF_SSL ("check-certificate"), 0, OPT_BOOLEAN, "checkcertificate", -1 },
{ "clobber", 0, OPT__CLOBBER, NULL, optional_argument },
+ { "config", 0, OPT_VALUE, "chooseconfig", -1 },
{ "connect-timeout", 0, OPT_VALUE, "connecttimeout", -1 },
{ "continue", 'c', OPT_BOOLEAN, "continue", -1 },
{ "convert-links", 'k', OPT_BOOLEAN, "convertlinks", -1 },
{ "content-disposition", 0, OPT_BOOLEAN, "contentdisposition", -1 },
+ { "content-on-error", 0, OPT_BOOLEAN, "contentonerror", -1 },
{ "cookies", 0, OPT_BOOLEAN, "cookies", -1 },
{ "cut-dirs", 0, OPT_VALUE, "cutdirs", -1 },
{ WHEN_DEBUG ("debug"), 'd', OPT_BOOLEAN, "debug", -1 },
{ "load-cookies", 0, OPT_VALUE, "loadcookies", -1 },
{ "local-encoding", 0, OPT_VALUE, "localencoding", -1 },
{ "max-redirect", 0, OPT_VALUE, "maxredirect", -1 },
+ { "method", 0, OPT_VALUE, "method", -1 },
{ "mirror", 'm', OPT_BOOLEAN, "mirror", -1 },
{ "no", 'n', OPT__NO, NULL, required_argument },
{ "no-clobber", 0, OPT_BOOLEAN, "noclobber", -1 },
{ "post-data", 0, OPT_VALUE, "postdata", -1 },
{ "post-file", 0, OPT_VALUE, "postfile", -1 },
{ "prefer-family", 0, OPT_VALUE, "preferfamily", -1 },
- { "preserve-permissions", 0, OPT_BOOLEAN, "preservepermissions", -1 }, /* deprecated */
+ { "preserve-permissions", 0, OPT_BOOLEAN, "preservepermissions", -1 },
{ IF_SSL ("private-key"), 0, OPT_VALUE, "privatekey", -1 },
{ IF_SSL ("private-key-type"), 0, OPT_VALUE, "privatekeytype", -1 },
{ "progress", 0, OPT_VALUE, "progress", -1 },
{ "read-timeout", 0, OPT_VALUE, "readtimeout", -1 },
{ "recursive", 'r', OPT_BOOLEAN, "recursive", -1 },
{ "referer", 0, OPT_VALUE, "referer", -1 },
+ { "regex-type", 0, OPT_VALUE, "regextype", -1 },
{ "reject", 'R', OPT_VALUE, "reject", -1 },
+ { "reject-regex", 0, OPT_VALUE, "rejectregex", -1 },
{ "relative", 'L', OPT_BOOLEAN, "relativeonly", -1 },
{ "remote-encoding", 0, OPT_VALUE, "remoteencoding", -1 },
{ "remove-listing", 0, OPT_BOOLEAN, "removelisting", -1 },
+ { "report-speed", 0, OPT_BOOLEAN, "reportspeed", -1 },
{ "restrict-file-names", 0, OPT_BOOLEAN, "restrictfilenames", -1 },
{ "retr-symlinks", 0, OPT_BOOLEAN, "retrsymlinks", -1 },
{ "retry-connrefused", 0, OPT_BOOLEAN, "retryconnrefused", -1 },
{ "timeout", 'T', OPT_VALUE, "timeout", -1 },
{ "timestamping", 'N', OPT_BOOLEAN, "timestamping", -1 },
{ "tries", 't', OPT_VALUE, "tries", -1 },
+ { "unlink", 0, OPT_BOOLEAN, "unlink", -1 },
+ { "trust-server-names", 0, OPT_BOOLEAN, "trustservernames", -1 },
{ "use-server-timestamps", 0, OPT_BOOLEAN, "useservertimestamps", -1 },
{ "user", 0, OPT_VALUE, "user", -1 },
{ "user-agent", 'U', OPT_VALUE, "useragent", -1 },
{ "version", 'V', OPT_FUNCALL, (void *) print_version, no_argument },
{ "wait", 'w', OPT_VALUE, "wait", -1 },
{ "waitretry", 0, OPT_VALUE, "waitretry", -1 },
+ { "warc-cdx", 0, OPT_BOOLEAN, "warccdx", -1 },
+#ifdef HAVE_LIBZ
+ { "warc-compression", 0, OPT_BOOLEAN, "warccompression", -1 },
+#endif
+ { "warc-dedup", 0, OPT_VALUE, "warccdxdedup", -1 },
+ { "warc-digests", 0, OPT_BOOLEAN, "warcdigests", -1 },
+ { "warc-file", 0, OPT_VALUE, "warcfile", -1 },
+ { "warc-header", 0, OPT_VALUE, "warcheader", -1 },
+ { "warc-keep-log", 0, OPT_BOOLEAN, "warckeeplog", -1 },
+ { "warc-max-size", 0, OPT_VALUE, "warcmaxsize", -1 },
+ { "warc-tempdir", 0, OPT_VALUE, "warctempdir", -1 },
#ifdef USE_WATT32
{ "wdebug", 0, OPT_BOOLEAN, "wdebug", -1 },
#endif
}
/* Print the usage message. */
-static void
+static int
print_usage (int error)
{
- fprintf (error ? stderr : stdout, _("Usage: %s [OPTION]... [URL]...\n"),
- exec_name);
+ return fprintf (error ? stderr : stdout,
+ _("Usage: %s [OPTION]... [URL]...\n"), exec_name);
}
/* Print the help message, describing all the available options. If
-v, --verbose be verbose (this is the default).\n"),
N_("\
-nv, --no-verbose turn off verboseness, without being quiet.\n"),
+ N_("\
+ --report-speed=TYPE Output bandwidth as TYPE. TYPE can be bits.\n"),
N_("\
-i, --input-file=FILE download URLs found in local or external FILE.\n"),
N_("\
N_("\
-B, --base=URL resolves HTML input-file links (-i -F)\n\
relative to URL.\n"),
+ N_("\
+ --config=FILE Specify config file to use.\n"),
"\n",
N_("\
-O, --output-document=FILE write documents to FILE.\n"),
N_("\
-nc, --no-clobber skip downloads that would download to\n\
- existing files.\n"),
+ existing files (overwriting them).\n"),
N_("\
-c, --continue resume getting a partially-downloaded file.\n"),
N_("\
--local-encoding=ENC use ENC as the local encoding for IRIs.\n"),
N_("\
--remote-encoding=ENC use ENC as the default remote encoding.\n"),
+ N_("\
+ --unlink remove file before clobber.\n"),
"\n",
N_("\
--post-data=STRING use the POST method; send STRING as the data.\n"),
N_("\
--post-file=FILE use the POST method; send contents of FILE.\n"),
+ N_("\
+ --method=HTTPMethod use method \"HTTPMethod\" in the header.\n"),
+ N_("\
+ --body-data=STRING Send STRING as data. --method MUST be set.\n"),
+ N_("\
+ --body-file=FILE Send contents of FILE. --method MUST be set.\n"),
N_("\
--content-disposition honor the Content-Disposition header when\n\
choosing local file names (EXPERIMENTAL).\n"),
+ N_("\
+ --content-on-error output the received content on server errors.\n"),
N_("\
--auth-no-challenge send Basic HTTP authentication information\n\
without first waiting for the server's\n\
--no-glob turn off FTP file name globbing.\n"),
N_("\
--no-passive-ftp disable the \"passive\" transfer mode.\n"),
+ N_("\
+ --preserve-permissions preserve remote file permissions.\n"),
N_("\
--retr-symlinks when recursing, get linked-to files (not dir).\n"),
"\n",
+ N_("\
+WARC options:\n"),
+ N_("\
+ --warc-file=FILENAME save request/response data to a .warc.gz file.\n"),
+ N_("\
+ --warc-header=STRING insert STRING into the warcinfo record.\n"),
+ N_("\
+ --warc-max-size=NUMBER set maximum size of WARC files to NUMBER.\n"),
+ N_("\
+ --warc-cdx write CDX index files.\n"),
+ N_("\
+ --warc-dedup=FILENAME do not store records listed in this CDX file.\n"),
+#ifdef HAVE_LIBZ
+ N_("\
+ --no-warc-compression do not compress WARC files with GZIP.\n"),
+#endif
+ N_("\
+ --no-warc-digests do not calculate SHA1 digests.\n"),
+ N_("\
+ --no-warc-keep-log do not store the log file in a WARC record.\n"),
+ N_("\
+ --warc-tempdir=DIRECTORY location for temporary files created by the\n\
+ WARC writer.\n"),
+ "\n",
+
N_("\
Recursive download:\n"),
N_("\
N_("\
-k, --convert-links make links in downloaded HTML or CSS point to\n\
local files.\n"),
+ N_("\
+ --backups=N before writing file X, rotate up to N backup files.\n"),
+
#ifdef __VMS
N_("\
-K, --backup-converted before converting file X, back up as X_orig.\n"),
-A, --accept=LIST comma-separated list of accepted extensions.\n"),
N_("\
-R, --reject=LIST comma-separated list of rejected extensions.\n"),
+ N_("\
+ --accept-regex=REGEX regex matching accepted URLs.\n"),
+ N_("\
+ --reject-regex=REGEX regex matching rejected URLs.\n"),
+#ifdef HAVE_LIBPCRE
+ N_("\
+ --regex-type=TYPE regex type (posix|pcre).\n"),
+#else
+ N_("\
+ --regex-type=TYPE regex type (posix).\n"),
+#endif
N_("\
-D, --domains=LIST comma-separated list of accepted domains.\n"),
N_("\
N_("\
-I, --include-directories=LIST list of allowed directories.\n"),
N_("\
+ --trust-server-names use the name specified by the redirection\n\
+ url last component.\n"),
+ N_("\
-X, --exclude-directories=LIST list of excluded directories.\n"),
N_("\
-np, --no-parent don't ascend to the parent directory.\n"),
"\n",
-
N_("Mail bug reports and suggestions to <bug-wget@gnu.org>.\n")
};
size_t i;
- printf (_("GNU Wget %s, a non-interactive network retriever.\n"),
- version_string);
- print_usage (0);
+ if (printf (_("GNU Wget %s, a non-interactive network retriever.\n"),
+ version_string) < 0)
+ exit (3);
+ if (print_usage (0) < 0)
+ exit (3);
for (i = 0; i < countof (help); i++)
- fputs (_(help[i]), stdout);
+ if (fputs (_(help[i]), stdout) < 0)
+ exit (3);
exit (0);
}
prompt_for_password (void)
{
if (opt.user)
- printf (_("Password for user %s: "), quote (opt.user));
+ fprintf (stderr, _("Password for user %s: "), quote (opt.user));
else
- printf (_("Password: "));
+ fprintf (stderr, _("Password: "));
return getpass("");
}
to at most line_length. prefix is printed on the first line
and an appropriate number of spaces are added on subsequent
lines.*/
-static void
+static int
format_and_print_line (const char *prefix, const char *line,
int line_length)
{
if (line_length <= 0)
line_length = MAX_CHARS_PER_LINE - TABULATION;
- printf ("%s", prefix);
+ if (printf ("%s", prefix) < 0)
+ return -1;
remaining_chars = line_length;
/* We break on spaces. */
token = strtok (line_dup, " ");
token on the next line. */
if (remaining_chars <= strlen (token))
{
- printf ("\n%*c", TABULATION, ' ');
+ if (printf ("\n%*c", TABULATION, ' ') < 0)
+ return -1;
remaining_chars = line_length - TABULATION;
}
- printf ("%s ", token);
+ if (printf ("%s ", token) < 0)
+ return -1;
remaining_chars -= strlen (token) + 1; /* account for " " */
token = strtok (NULL, " ");
}
- printf ("\n");
+ if (printf ("\n") < 0)
+ return -1;
xfree (line_dup);
+ return 0;
}
static void
char *env_wgetrc, *user_wgetrc;
int i;
- printf (_("GNU Wget %s built on %s.\n\n"), version_string, OS_TYPE);
+ if (printf (_("GNU Wget %s built on %s.\n\n"), version_string, OS_TYPE) < 0)
+ exit (3);
for (i = 0; compiled_features[i] != NULL; )
{
int line_length = MAX_CHARS_PER_LINE;
while ((line_length > 0) && (compiled_features[i] != NULL))
{
- printf ("%s ", compiled_features[i]);
+ if (printf ("%s ", compiled_features[i]) < 0)
+ exit (3);
line_length -= strlen (compiled_features[i]) + 2;
i++;
}
- printf ("\n");
+ if (printf ("\n") < 0)
+ exit (3);
}
- printf ("\n");
+ if (printf ("\n") < 0)
+ exit (3);
/* Handle the case when $WGETRC is unset and $HOME/.wgetrc is
absent. */
- printf ("%s\n", wgetrc_title);
+ if (printf ("%s\n", wgetrc_title) < 0)
+ exit (3);
+
env_wgetrc = wgetrc_env_file_name ();
if (env_wgetrc && *env_wgetrc)
{
- printf (_(" %s (env)\n"), env_wgetrc);
+ if (printf (_(" %s (env)\n"), env_wgetrc) < 0)
+ exit (3);
xfree (env_wgetrc);
}
user_wgetrc = wgetrc_user_file_name ();
if (user_wgetrc)
{
- printf (_(" %s (user)\n"), user_wgetrc);
+ if (printf (_(" %s (user)\n"), user_wgetrc) < 0)
+ exit (3);
xfree (user_wgetrc);
}
#ifdef SYSTEM_WGETRC
- printf (_(" %s (system)\n"), SYSTEM_WGETRC);
+ if (printf (_(" %s (system)\n"), SYSTEM_WGETRC) < 0)
+ exit (3);
#endif
#ifdef ENABLE_NLS
- format_and_print_line (locale_title,
+ if (format_and_print_line (locale_title,
LOCALEDIR,
- MAX_CHARS_PER_LINE);
+ MAX_CHARS_PER_LINE) < 0)
+ exit (3);
#endif /* def ENABLE_NLS */
if (compilation_string != NULL)
- format_and_print_line (compile_title,
- compilation_string,
- MAX_CHARS_PER_LINE);
+ if (format_and_print_line (compile_title,
+ compilation_string,
+ MAX_CHARS_PER_LINE) < 0)
+ exit (3);
if (link_string != NULL)
- format_and_print_line (link_title,
- link_string,
- MAX_CHARS_PER_LINE);
+ if (format_and_print_line (link_title,
+ link_string,
+ MAX_CHARS_PER_LINE) < 0)
+ exit (3);
+
+ if (printf ("\n") < 0)
+ exit (3);
- printf ("\n");
/* TRANSLATORS: When available, an actual copyright character
- (cirle-c) should be used in preference to "(C)". */
- fputs (_("\
-Copyright (C) 2009 Free Software Foundation, Inc.\n"), stdout);
- fputs (_("\
+ (circle-c) should be used in preference to "(C)". */
+ if (fputs (_("\
+Copyright (C) 2011 Free Software Foundation, Inc.\n"), stdout) < 0)
+ exit (3);
+ if (fputs (_("\
License GPLv3+: GNU GPL version 3 or later\n\
<http://www.gnu.org/licenses/gpl.html>.\n\
This is free software: you are free to change and redistribute it.\n\
-There is NO WARRANTY, to the extent permitted by law.\n"), stdout);
+There is NO WARRANTY, to the extent permitted by law.\n"), stdout) < 0)
+ exit (3);
/* TRANSLATORS: When available, please use the proper diacritics for
names such as this one. See en_US.po for reference. */
- fputs (_("\nOriginally written by Hrvoje Niksic <hniksic@xemacs.org>.\n"),
- stdout);
- fputs (_("Please send bug reports and questions to <bug-wget@gnu.org>.\n"),
- stdout);
+ if (fputs (_("\nOriginally written by Hrvoje Niksic <hniksic@xemacs.org>.\n"),
+ stdout) < 0)
+ exit (3);
+ if (fputs (_("Please send bug reports and questions to <bug-wget@gnu.org>.\n"),
+ stdout) < 0)
+ exit (3);
+
exit (0);
}
char *program_name; /* Needed by lib/error.c. */
+char *program_argstring; /* Needed by wget_warc.c. */
int
main (int argc, char **argv)
{
char **url, **t;
int i, ret, longindex;
- int nurl, status;
+ int nurl;
bool append_to_log = false;
+ total_downloaded_bytes = 0;
+
program_name = argv[0];
+ struct ptimer *timer = ptimer_new ();
+ double start_time = ptimer_measure (timer);
+
i18n_initialize ();
/* Construct the name of the executable, without the directory part. */
windows_main ((char **) &exec_name);
#endif
- /* Set option defaults; read the system wgetrc and ~/.wgetrc. */
- initialize ();
+ /* Construct the arguments string. */
+ int argstring_length = 1;
+ for (i = 1; i < argc; i++)
+ argstring_length += strlen (argv[i]) + 2 + 1;
+ char *p = program_argstring = malloc (argstring_length * sizeof (char));
+ if (p == NULL)
+ {
+ fprintf (stderr, _("Memory allocation problem\n"));
+ exit (2);
+ }
+ for (i = 1; i < argc; i++)
+ {
+ *p++ = '"';
+ int arglen = strlen (argv[i]);
+ memcpy (p, argv[i], arglen);
+ p += arglen;
+ *p++ = '"';
+ *p++ = ' ';
+ }
+ *p = '\0';
+
+ /* Load the hard-coded defaults. */
+ defaults ();
init_switches ();
+
+ /* This separate getopt_long is needed to find the user config file
+ option ("--config") and parse it before the other user options. */
+ longindex = -1;
+ int retconf;
+ bool use_userconfig = false;
+
+ while ((retconf = getopt_long (argc, argv,
+ short_options, long_options, &longindex)) != -1)
+ {
+ int confval;
+ struct cmdline_option *config_opt;
+
+ /* There is no short option for "--config". */
+ if (longindex >= 0)
+ {
+ confval = long_options[longindex].val;
+ config_opt = &option_data[confval & ~BOOLEAN_NEG_MARKER];
+ if (strcmp (config_opt->long_name, "config") == 0)
+ {
+ bool userrc_ret = true;
+ userrc_ret &= run_wgetrc (optarg);
+ use_userconfig = true;
+ if (userrc_ret)
+ break;
+ else
+ {
+ fprintf (stderr, _("Exiting due to error in %s\n"), optarg);
+ exit (2);
+ }
+ }
+ }
+ }
+
+ /* If the user did not specify a config, read the system wgetrc and ~/.wgetrc. */
+ if (use_userconfig == false)
+ initialize ();
+
+ opterr = 0;
+ optind = 0;
+
longindex = -1;
while ((ret = getopt_long (argc, argv,
short_options, long_options, &longindex)) != -1)
{
if (ret == '?')
{
- print_usage (0);
- printf ("\n");
- printf (_("Try `%s --help' for more options.\n"), exec_name);
+ print_usage (1);
+ fprintf (stderr, "\n");
+ fprintf (stderr, _("Try `%s --help' for more options.\n"),
+ exec_name);
exit (2);
}
/* Find the short option character in the mapping. */
short options for convenience and backward
compatibility. */
char *p;
- for (p = optarg; *p; p++)
+ for (p = optarg; p && *p; p++)
switch (*p)
{
case 'v':
/* All user options have now been processed, so it's now safe to do
interoption dependency checks. */
+ if (opt.noclobber && opt.convert_links)
+ {
+ fprintf (stderr,
+ _("Both --no-clobber and --convert-links were specified,"
+ " only --convert-links will be used.\n"));
+ opt.noclobber = false;
+ }
+
if (opt.reclevel == 0)
opt.reclevel = INFINITE_RECURSION; /* see recur.h for commentary */
if (opt.verbose == -1)
opt.verbose = !opt.quiet;
+
/* Sanity checks. */
if (opt.verbose && opt.quiet)
{
}
}
+ if (opt.warc_filename != 0)
+ {
+ if (opt.noclobber)
+ {
+ fprintf (stderr,
+ _("WARC output does not work with --no-clobber, "
+ "--no-clobber will be disabled.\n"));
+ opt.noclobber = false;
+ }
+ if (opt.timestamping)
+ {
+ fprintf (stderr,
+ _("WARC output does not work with timestamping, "
+ "timestamping will be disabled.\n"));
+ opt.timestamping = false;
+ }
+ if (opt.spider)
+ {
+ fprintf (stderr,
+ _("WARC output does not work with --spider.\n"));
+ exit (1);
+ }
+ if (opt.always_rest)
+ {
+ fprintf (stderr,
+ _("WARC output does not work with --continue, "
+ "--continue will be disabled.\n"));
+ opt.always_rest = false;
+ }
+ if (opt.warc_cdx_dedup_filename != 0 && !opt.warc_digests_enabled)
+ {
+ fprintf (stderr,
+ _("Digests are disabled; WARC deduplication will "
+ "not find duplicate records.\n"));
+ }
+ if (opt.warc_keep_log)
+ {
+ opt.progress_type = xstrdup ("dot");
+ }
+ }
+
if (opt.ask_passwd && opt.passwd)
{
fprintf (stderr,
/* No URL specified. */
fprintf (stderr, _("%s: missing URL\n"), exec_name);
print_usage (1);
- printf ("\n");
+ fprintf (stderr, "\n");
/* #### Something nicer should be printed here -- similar to the
pre-1.5 `--help' page. */
fprintf (stderr, _("Try `%s --help' for more options.\n"), exec_name);
exit (1);
}
+ /* Compile the regular expressions. */
+ switch (opt.regex_type)
+ {
+#ifdef HAVE_LIBPCRE
+ case regex_type_pcre:
+ opt.regex_compile_fun = compile_pcre_regex;
+ opt.regex_match_fun = match_pcre_regex;
+ break;
+#endif
+
+ case regex_type_posix:
+ default:
+ opt.regex_compile_fun = compile_posix_regex;
+ opt.regex_match_fun = match_posix_regex;
+ break;
+ }
+ if (opt.acceptregex_s)
+ {
+ opt.acceptregex = opt.regex_compile_fun (opt.acceptregex_s);
+ if (!opt.acceptregex)
+ exit (1);
+ }
+ if (opt.rejectregex_s)
+ {
+ opt.rejectregex = opt.regex_compile_fun (opt.rejectregex_s);
+ if (!opt.rejectregex)
+ exit (1);
+ }
+ if (opt.post_data || opt.post_file_name)
+ {
+ if (opt.post_data && opt.post_file_name)
+ {
+ fprintf (stderr, _("You cannot specify both --post-data and --post-file.\n"));
+ exit (1);
+ }
+ else if (opt.method)
+ {
+ fprintf (stderr, _("You cannot use --post-data or --post-file along with --method. "
+ "--method expects data through --body-data and --body-file options"));
+ exit (1);
+ }
+ }
+ if (opt.body_data || opt.body_file)
+ {
+ if (!opt.method)
+ {
+ fprintf (stderr, _("You must specify a method through --method=HTTPMethod "
+ "to use with --body-data or --body-file.\n"));
+ exit (1);
+ }
+ else if (opt.body_data && opt.body_file)
+ {
+ fprintf (stderr, _("You cannot specify both --body-data and --body-file.\n"));
+ exit (1);
+ }
+ }
+
+ /* Set various options as required for opt.method. */
+
+ /* When user specifies HEAD as the method, we do not wish to download any
+ files. Hence, set wget to run in spider mode. */
+ if (opt.method && strcasecmp (opt.method, "HEAD") == 0)
+ setoptval ("spider", "1", "spider");
+
+ /* Convert post_data to body-data and post_file_name to body-file options.
+ This is required so as to remove redundant code later on in gethttp().
+ The --post-data and --post-file options may also be removed in
+ the future hence it makes sense to convert them to aliases for
+ the more generic --method options.
+ This MUST occur only after the sanity checks so as to prevent the
+ user from setting both post and body options simultaneously.
+ */
+ if (opt.post_data || opt.post_file_name)
+ {
+ setoptval ("method", "POST", "method");
+ if (opt.post_data)
+ {
+ setoptval ("bodydata", opt.post_data, "body-data");
+ opt.post_data = NULL;
+ }
+ else
+ {
+ setoptval ("bodyfile", opt.post_file_name, "body-file");
+ opt.post_file_name = NULL;
+ }
+ }
+
#ifdef ENABLE_IRI
if (opt.enable_iri)
{
opt.encoding_remote = NULL;
}
#else
+ memset (&dummy_iri, 0, sizeof (dummy_iri));
if (opt.enable_iri || opt.locale || opt.encoding_remote)
{
/* sXXXav : be more specific... */
/* Fill in the arguments. */
url = alloca_array (char *, nurl + 1);
+ if (url == NULL)
+ {
+ fprintf (stderr, _("Memory allocation problem\n"));
+ exit (2);
+ }
for (i = 0; i < nurl; i++, optind++)
{
char *rewritten = rewrite_shorthand_url (argv[optind]);
/* Initialize logging. */
log_init (opt.lfilename, append_to_log);
+ /* Open WARC file. */
+ if (opt.warc_filename != 0)
+ warc_init ();
+
DEBUGP (("DEBUG output created by Wget %s on %s.\n\n",
version_string, OS_TYPE));
if (HYPHENP (opt.output_document))
{
#ifdef WINDOWS
- FILE *result;
- result = freopen ("CONOUT$", "wb", stdout);
- if (result == NULL)
- {
- logputs (LOG_NOTQUIET, _("\
-WARNING: Can't reopen standard output in binary mode;\n\
- downloaded file may contain inappropriate line endings.\n"));
- }
+ _setmode (_fileno (stdout), _O_BINARY);
#endif
output_stream = stdout;
}
signal (SIGWINCH, progress_handle_sigwinch);
#endif
- status = RETROK; /* initialize it, just-in-case */
/* Retrieve the URLs from argument list. */
for (t = url; *t; t++)
{
char *error = url_error (*t, url_err);
logprintf (LOG_NOTQUIET, "%s: %s.\n",*t, error);
xfree (error);
- status = URLERROR;
+ inform_exit_status (URLERROR);
}
else
{
if (url_scheme (*t) == SCHEME_FTP)
opt.follow_ftp = 1;
- status = retrieve_tree (url_parsed, NULL);
+ retrieve_tree (url_parsed, NULL);
opt.follow_ftp = old_follow_ftp;
}
else
{
- status = retrieve_url (url_parsed, *t, &filename, &redirected_URL,
- NULL, &dt, opt.recursive, iri, true);
+ retrieve_url (url_parsed, *t, &filename, &redirected_URL, NULL,
+ &dt, opt.recursive, iri, true);
}
- if (opt.delete_after && file_exists_p(filename))
+ if (opt.delete_after && filename != NULL && file_exists_p (filename))
{
DEBUGP (("Removing file due to --delete-after in main():\n"));
logprintf (LOG_VERBOSE, _("Removing %s.\n"), filename);
if (opt.input_filename)
{
int count;
+ int status;
status = retrieve_from_file (opt.input_filename, opt.force_html, &count);
+ inform_exit_status (status);
if (!count)
logprintf (LOG_NOTQUIET, _("No URLs found in %s.\n"),
opt.input_filename);
&&
total_downloaded_bytes != 0)
{
+ double end_time = ptimer_measure (timer);
+ ptimer_destroy (timer);
+
+ char *wall_time = xstrdup (secs_to_human_time (end_time - start_time));
+ char *download_time = xstrdup (secs_to_human_time (total_download_time));
logprintf (LOG_NOTQUIET,
- _("FINISHED --%s--\nDownloaded: %d files, %s in %s (%s)\n"),
- datetime_str (time (NULL)),
- numurls,
- human_readable (total_downloaded_bytes),
- secs_to_human_time (total_download_time),
- retr_rate (total_downloaded_bytes, total_download_time));
+ _("FINISHED --%s--\nTotal wall clock time: %s\n"
+ "Downloaded: %d files, %s in %s (%s)\n"),
+ datetime_str (time (NULL)),
+ wall_time,
+ numurls,
+ human_readable (total_downloaded_bytes),
+ download_time,
+ retr_rate (total_downloaded_bytes, total_download_time));
+ xfree (wall_time);
+ xfree (download_time);
+
/* Print quota warning, if exceeded. */
if (opt.quota && total_downloaded_bytes > opt.quota)
logprintf (LOG_NOTQUIET,
if (opt.convert_links && !opt.delete_after)
convert_all_links ();
- log_close ();
- for (i = 0; i < nurl; i++)
- xfree (url[i]);
cleanup ();
- return get_exit_status ();
+ exit (get_exit_status ());
}
#endif /* TESTING */
\f