X-Git-Url: http://sjero.net/git/?p=wget;a=blobdiff_plain;f=src%2Fmain.c;h=29c13242834389638ee906f96d0d6eb439920154;hp=3a241fbbf3739e408d3677a098882a7b9d8decd9;hb=38a7829dcb4eb5dba28dbf0f05c6a80fea9217f8;hpb=a6625c97c57b15d77d58545019d951085c3a3d37 diff --git a/src/main.c b/src/main.c index 3a241fbb..29c13242 100644 --- a/src/main.c +++ b/src/main.c @@ -1,6 +1,7 @@ /* Command line parsing. Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, - 2005, 2006, 2007, 2008, 2009, 2010 Free Software Foundation, Inc. + 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 Free Software Foundation, + Inc. This file is part of GNU Wget. @@ -32,9 +33,7 @@ as that of the covered work. */ #include #include -#ifdef HAVE_UNISTD_H -# include -#endif /* HAVE_UNISTD_H */ +#include #include #include #ifdef ENABLE_NLS @@ -55,19 +54,29 @@ as that of the covered work. */ #include "convert.h" #include "spider.h" #include "http.h" /* for save_cookies */ - +#include "ptimer.h" +#include "warc.h" #include #include #include +#ifdef WINDOWS +# include +# include +#endif + #ifdef __VMS -#include "vms.h" +# include "vms.h" #endif /* __VMS */ #ifndef PATH_SEPARATOR # define PATH_SEPARATOR '/' #endif +#ifndef ENABLE_IRI +struct iri dummy_iri; +#endif + struct options opt; /* defined in version.c */ @@ -119,12 +128,6 @@ static void print_version (void); # define IF_SSL(x) NULL #endif -#ifdef ENABLE_DEBUG -# define WHEN_DEBUG(x) x -#else -# define WHEN_DEBUG(x) NULL -#endif - struct cmdline_option { const char *long_name; char short_name; @@ -148,6 +151,7 @@ struct cmdline_option { static struct cmdline_option option_data[] = { { "accept", 'A', OPT_VALUE, "accept", -1 }, + { "accept-regex", 0, OPT_VALUE, "acceptregex", -1 }, { "adjust-extension", 'E', OPT_BOOLEAN, "adjustextension", -1 }, { "append-output", 'a', OPT__APPEND_OUTPUT, NULL, required_argument }, { "ask-password", 0, OPT_BOOLEAN, "askpassword", -1 }, @@ -157,6 +161,8 @@ static struct cmdline_option option_data[] = { "backups", 0, OPT_BOOLEAN, "backups", -1 }, { "base", 'B', OPT_VALUE, "base", -1 }, { "bind-address", 0, OPT_VALUE, "bindaddress", -1 }, + { "body-data", 0, OPT_VALUE, "bodydata", -1 }, + { "body-file", 0, OPT_VALUE, "bodyfile", -1 }, { IF_SSL ("ca-certificate"), 0, OPT_VALUE, "cacertificate", -1 }, { IF_SSL ("ca-directory"), 0, OPT_VALUE, "cadirectory", -1 }, { "cache", 0, OPT_BOOLEAN, "cache", -1 }, @@ -164,13 +170,15 @@ static struct cmdline_option option_data[] = { IF_SSL ("certificate-type"), 0, OPT_VALUE, "certificatetype", -1 }, { IF_SSL ("check-certificate"), 0, OPT_BOOLEAN, "checkcertificate", -1 }, { "clobber", 0, OPT__CLOBBER, NULL, optional_argument }, + { "config", 0, OPT_VALUE, "chooseconfig", -1 }, { "connect-timeout", 0, OPT_VALUE, "connecttimeout", -1 }, { "continue", 'c', OPT_BOOLEAN, "continue", -1 }, { "convert-links", 'k', OPT_BOOLEAN, "convertlinks", -1 }, { "content-disposition", 0, OPT_BOOLEAN, "contentdisposition", -1 }, + { "content-on-error", 0, OPT_BOOLEAN, "contentonerror", -1 }, { "cookies", 0, OPT_BOOLEAN, "cookies", -1 }, { "cut-dirs", 0, OPT_VALUE, "cutdirs", -1 }, - { WHEN_DEBUG ("debug"), 'd', OPT_BOOLEAN, "debug", -1 }, + { "debug", 'd', OPT_BOOLEAN, "debug", -1 }, { "default-page", 0, OPT_VALUE, "defaultpage", -1 }, { "delete-after", 0, OPT_BOOLEAN, "deleteafter", -1 }, { "directories", 0, OPT_BOOLEAN, "dirstruct", -1 }, @@ -203,6 +211,7 @@ static struct cmdline_option option_data[] = { "http-passwd", 0, OPT_VALUE, "httppassword", -1 }, /* deprecated */ { "http-password", 0, OPT_VALUE, "httppassword", -1 }, { "http-user", 0, OPT_VALUE, "httpuser", -1 }, + { IF_SSL ("https-only"), 0, OPT_BOOLEAN, "httpsonly", -1 }, { "ignore-case", 0, OPT_BOOLEAN, "ignorecase", -1 }, { "ignore-length", 0, OPT_BOOLEAN, "ignorelength", -1 }, { "ignore-tags", 0, OPT_VALUE, "ignoretags", -1 }, @@ -219,9 +228,11 @@ static struct cmdline_option option_data[] = { "load-cookies", 0, OPT_VALUE, "loadcookies", -1 }, { "local-encoding", 0, OPT_VALUE, "localencoding", -1 }, { "max-redirect", 0, OPT_VALUE, "maxredirect", -1 }, + { "method", 0, OPT_VALUE, "method", -1 }, { "mirror", 'm', OPT_BOOLEAN, "mirror", -1 }, { "no", 'n', OPT__NO, NULL, required_argument }, { "no-clobber", 0, OPT_BOOLEAN, "noclobber", -1 }, + { "no-config", 0, OPT_BOOLEAN, "noconfig", -1}, { "no-parent", 0, OPT_BOOLEAN, "noparent", -1 }, { "output-document", 'O', OPT_VALUE, "outputdocument", -1 }, { "output-file", 'o', OPT_VALUE, "logfile", -1 }, @@ -232,10 +243,11 @@ static struct cmdline_option option_data[] = { "post-data", 0, OPT_VALUE, "postdata", -1 }, { "post-file", 0, OPT_VALUE, "postfile", -1 }, { "prefer-family", 0, OPT_VALUE, "preferfamily", -1 }, - { "preserve-permissions", 0, OPT_BOOLEAN, "preservepermissions", -1 }, /* deprecated */ + { "preserve-permissions", 0, OPT_BOOLEAN, "preservepermissions", -1 }, { IF_SSL ("private-key"), 0, OPT_VALUE, "privatekey", -1 }, { IF_SSL ("private-key-type"), 0, OPT_VALUE, "privatekeytype", -1 }, { "progress", 0, OPT_VALUE, "progress", -1 }, + { "show-progress", 0, OPT_BOOLEAN, "showprogress", -1 }, { "protocol-directories", 0, OPT_BOOLEAN, "protocoldirectories", -1 }, { "proxy", 0, OPT_BOOLEAN, "useproxy", -1 }, { "proxy__compat", 'Y', OPT_VALUE, "useproxy", -1 }, /* back-compatible */ @@ -249,10 +261,13 @@ static struct cmdline_option option_data[] = { "read-timeout", 0, OPT_VALUE, "readtimeout", -1 }, { "recursive", 'r', OPT_BOOLEAN, "recursive", -1 }, { "referer", 0, OPT_VALUE, "referer", -1 }, + { "regex-type", 0, OPT_VALUE, "regextype", -1 }, { "reject", 'R', OPT_VALUE, "reject", -1 }, + { "reject-regex", 0, OPT_VALUE, "rejectregex", -1 }, { "relative", 'L', OPT_BOOLEAN, "relativeonly", -1 }, { "remote-encoding", 0, OPT_VALUE, "remoteencoding", -1 }, { "remove-listing", 0, OPT_BOOLEAN, "removelisting", -1 }, + { "report-speed", 0, OPT_BOOLEAN, "reportspeed", -1 }, { "restrict-file-names", 0, OPT_BOOLEAN, "restrictfilenames", -1 }, { "retr-symlinks", 0, OPT_BOOLEAN, "retrsymlinks", -1 }, { "retry-connrefused", 0, OPT_BOOLEAN, "retryconnrefused", -1 }, @@ -262,10 +277,13 @@ static struct cmdline_option option_data[] = { "server-response", 'S', OPT_BOOLEAN, "serverresponse", -1 }, { "span-hosts", 'H', OPT_BOOLEAN, "spanhosts", -1 }, { "spider", 0, OPT_BOOLEAN, "spider", -1 }, + { "start-pos", 0, OPT_VALUE, "startpos", -1 }, { "strict-comments", 0, OPT_BOOLEAN, "strictcomments", -1 }, { "timeout", 'T', OPT_VALUE, "timeout", -1 }, { "timestamping", 'N', OPT_BOOLEAN, "timestamping", -1 }, { "tries", 't', OPT_VALUE, "tries", -1 }, + { "unlink", 0, OPT_BOOLEAN, "unlink", -1 }, + { "trust-server-names", 0, OPT_BOOLEAN, "trustservernames", -1 }, { "use-server-timestamps", 0, OPT_BOOLEAN, "useservertimestamps", -1 }, { "user", 0, OPT_VALUE, "user", -1 }, { "user-agent", 'U', OPT_VALUE, "useragent", -1 }, @@ -274,12 +292,22 @@ static struct cmdline_option option_data[] = { "version", 'V', OPT_FUNCALL, (void *) print_version, no_argument }, { "wait", 'w', OPT_VALUE, "wait", -1 }, { "waitretry", 0, OPT_VALUE, "waitretry", -1 }, + { "warc-cdx", 0, OPT_BOOLEAN, "warccdx", -1 }, +#ifdef HAVE_LIBZ + { "warc-compression", 0, OPT_BOOLEAN, "warccompression", -1 }, +#endif + { "warc-dedup", 0, OPT_VALUE, "warccdxdedup", -1 }, + { "warc-digests", 0, OPT_BOOLEAN, "warcdigests", -1 }, + { "warc-file", 0, OPT_VALUE, "warcfile", -1 }, + { "warc-header", 0, OPT_VALUE, "warcheader", -1 }, + { "warc-keep-log", 0, OPT_BOOLEAN, "warckeeplog", -1 }, + { "warc-max-size", 0, OPT_VALUE, "warcmaxsize", -1 }, + { "warc-tempdir", 0, OPT_VALUE, "warctempdir", -1 }, #ifdef USE_WATT32 { "wdebug", 0, OPT_BOOLEAN, "wdebug", -1 }, #endif }; -#undef WHEN_DEBUG #undef IF_SSL /* Return a string that contains S with "no-" prepended. The string @@ -289,7 +317,7 @@ static struct cmdline_option option_data[] = static char * no_prefix (const char *s) { - static char buffer[1024]; + static char buffer[2048]; static char *p = buffer; char *cp = p; @@ -323,26 +351,26 @@ init_switches (void) size_t i, o = 0; for (i = 0; i < countof (option_data); i++) { - struct cmdline_option *opt = &option_data[i]; + struct cmdline_option *cmdopt = &option_data[i]; struct option *longopt; - if (!opt->long_name) + if (!cmdopt->long_name) /* The option is disabled. */ continue; longopt = &long_options[o++]; - longopt->name = opt->long_name; + longopt->name = cmdopt->long_name; longopt->val = i; - if (opt->short_name) + if (cmdopt->short_name) { - *p++ = opt->short_name; - optmap[opt->short_name - 32] = longopt - long_options; + *p++ = cmdopt->short_name; + optmap[cmdopt->short_name - 32] = longopt - long_options; } - switch (opt->type) + switch (cmdopt->type) { case OPT_VALUE: longopt->has_arg = required_argument; - if (opt->short_name) + if (cmdopt->short_name) *p++ = ':'; break; case OPT_BOOLEAN: @@ -356,16 +384,16 @@ init_switches (void) identical to "--foo", except it has opposite meaning and it doesn't allow an argument. */ longopt = &long_options[o++]; - longopt->name = no_prefix (opt->long_name); + longopt->name = no_prefix (cmdopt->long_name); longopt->has_arg = no_argument; /* Mask the value so we'll be able to recognize that we're dealing with the false value. */ longopt->val = i | BOOLEAN_NEG_MARKER; break; default: - assert (opt->argtype != -1); - longopt->has_arg = opt->argtype; - if (opt->short_name) + assert (cmdopt->argtype != -1); + longopt->has_arg = cmdopt->argtype; + if (cmdopt->short_name) { if (longopt->has_arg == required_argument) *p++ = ':'; @@ -381,11 +409,11 @@ init_switches (void) } /* Print the usage message. */ -static void +static int print_usage (int error) { - fprintf (error ? stderr : stdout, _("Usage: %s [OPTION]... [URL]...\n"), - exec_name); + return fprintf (error ? stderr : stdout, + _("Usage: %s [OPTION]... [URL]...\n"), exec_name); } /* Print the help message, describing all the available options. If @@ -431,6 +459,8 @@ Logging and input file:\n"), -v, --verbose be verbose (this is the default).\n"), N_("\ -nv, --no-verbose turn off verboseness, without being quiet.\n"), + N_("\ + --report-speed=TYPE Output bandwidth as TYPE. TYPE can be bits.\n"), N_("\ -i, --input-file=FILE download URLs found in local or external FILE.\n"), N_("\ @@ -438,6 +468,10 @@ Logging and input file:\n"), N_("\ -B, --base=URL resolves HTML input-file links (-i -F)\n\ relative to URL.\n"), + N_("\ + --config=FILE Specify config file to use.\n"), + N_("\ + --no-config Do not read any config file.\n"), "\n", N_("\ @@ -450,11 +484,15 @@ Download:\n"), -O, --output-document=FILE write documents to FILE.\n"), N_("\ -nc, --no-clobber skip downloads that would download to\n\ - existing files.\n"), + existing files (overwriting them).\n"), N_("\ -c, --continue resume getting a partially-downloaded file.\n"), + N_("\ + --start-pos=OFFSET start downloading from zero-based position OFFSET.\n"), N_("\ --progress=TYPE select progress gauge type.\n"), + N_("\ + --show-progress display the progress bar in any verbosity mode.\n"), N_("\ -N, --timestamping don't re-retrieve files unless newer than\n\ local.\n"), @@ -514,6 +552,8 @@ Download:\n"), --local-encoding=ENC use ENC as the local encoding for IRIs.\n"), N_("\ --remote-encoding=ENC use ENC as the default remote encoding.\n"), + N_("\ + --unlink remove file before clobber.\n"), "\n", N_("\ @@ -575,9 +615,17 @@ HTTP options:\n"), --post-data=STRING use the POST method; send STRING as the data.\n"), N_("\ --post-file=FILE use the POST method; send contents of FILE.\n"), + N_("\ + --method=HTTPMethod use method \"HTTPMethod\" in the header.\n"), + N_("\ + --body-data=STRING Send STRING as data. --method MUST be set.\n"), + N_("\ + --body-file=FILE Send contents of FILE. --method MUST be set.\n"), N_("\ --content-disposition honor the Content-Disposition header when\n\ choosing local file names (EXPERIMENTAL).\n"), + N_("\ + --content-on-error output the received content on server errors.\n"), N_("\ --auth-no-challenge send Basic HTTP authentication information\n\ without first waiting for the server's\n\ @@ -589,7 +637,9 @@ HTTP options:\n"), HTTPS (SSL/TLS) options:\n"), N_("\ --secure-protocol=PR choose secure protocol, one of auto, SSLv2,\n\ - SSLv3, and TLSv1.\n"), + SSLv3, TLSv1 and PFS.\n"), + N_("\ + --https-only only follow secure HTTPS links\n"), N_("\ --no-check-certificate don't validate the server's certificate.\n"), N_("\ @@ -627,10 +677,37 @@ FTP options:\n"), --no-glob turn off FTP file name globbing.\n"), N_("\ --no-passive-ftp disable the \"passive\" transfer mode.\n"), + N_("\ + --preserve-permissions preserve remote file permissions.\n"), N_("\ --retr-symlinks when recursing, get linked-to files (not dir).\n"), "\n", + N_("\ +WARC options:\n"), + N_("\ + --warc-file=FILENAME save request/response data to a .warc.gz file.\n"), + N_("\ + --warc-header=STRING insert STRING into the warcinfo record.\n"), + N_("\ + --warc-max-size=NUMBER set maximum size of WARC files to NUMBER.\n"), + N_("\ + --warc-cdx write CDX index files.\n"), + N_("\ + --warc-dedup=FILENAME do not store records listed in this CDX file.\n"), +#ifdef HAVE_LIBZ + N_("\ + --no-warc-compression do not compress WARC files with GZIP.\n"), +#endif + N_("\ + --no-warc-digests do not calculate SHA1 digests.\n"), + N_("\ + --no-warc-keep-log do not store the log file in a WARC record.\n"), + N_("\ + --warc-tempdir=DIRECTORY location for temporary files created by the\n\ + WARC writer.\n"), + "\n", + N_("\ Recursive download:\n"), N_("\ @@ -642,6 +719,9 @@ Recursive download:\n"), N_("\ -k, --convert-links make links in downloaded HTML or CSS point to\n\ local files.\n"), + N_("\ + --backups=N before writing file X, rotate up to N backup files.\n"), + #ifdef __VMS N_("\ -K, --backup-converted before converting file X, back up as X_orig.\n"), @@ -663,6 +743,17 @@ Recursive accept/reject:\n"), -A, --accept=LIST comma-separated list of accepted extensions.\n"), N_("\ -R, --reject=LIST comma-separated list of rejected extensions.\n"), + N_("\ + --accept-regex=REGEX regex matching accepted URLs.\n"), + N_("\ + --reject-regex=REGEX regex matching rejected URLs.\n"), +#ifdef HAVE_LIBPCRE + N_("\ + --regex-type=TYPE regex type (posix|pcre).\n"), +#else + N_("\ + --regex-type=TYPE regex type (posix).\n"), +#endif N_("\ -D, --domains=LIST comma-separated list of accepted domains.\n"), N_("\ @@ -680,22 +771,27 @@ Recursive accept/reject:\n"), N_("\ -I, --include-directories=LIST list of allowed directories.\n"), N_("\ + --trust-server-names use the name specified by the redirection\n\ + url last component.\n"), + N_("\ -X, --exclude-directories=LIST list of excluded directories.\n"), N_("\ -np, --no-parent don't ascend to the parent directory.\n"), "\n", - N_("Mail bug reports and suggestions to .\n") }; size_t i; - printf (_("GNU Wget %s, a non-interactive network retriever.\n"), - version_string); - print_usage (0); + if (printf (_("GNU Wget %s, a non-interactive network retriever.\n"), + version_string) < 0) + exit (3); + if (print_usage (0) < 0) + exit (3); for (i = 0; i < countof (help); i++) - fputs (_(help[i]), stdout); + if (fputs (_(help[i]), stdout) < 0) + exit (3); exit (0); } @@ -730,9 +826,9 @@ static char * prompt_for_password (void) { if (opt.user) - printf (_("Password for user %s: "), quote (opt.user)); + fprintf (stderr, _("Password for user %s: "), quote (opt.user)); else - printf (_("Password: ")); + fprintf (stderr, _("Password: ")); return getpass(""); } @@ -740,7 +836,7 @@ prompt_for_password (void) to at most line_length. prefix is printed on the first line and an appropriate number of spaces are added on subsequent lines.*/ -static void +static int format_and_print_line (const char *prefix, const char *line, int line_length) { @@ -749,14 +845,16 @@ format_and_print_line (const char *prefix, const char *line, assert (prefix != NULL); assert (line != NULL); + assert (line_length > TABULATION); line_dup = xstrdup (line); - if (line_length <= 0) - line_length = MAX_CHARS_PER_LINE - TABULATION; + if (printf ("%s", prefix) < 0) + return -1; + + /* Wrap to new line after prefix. */ + remaining_chars = 0; - printf ("%s", prefix); - remaining_chars = line_length; /* We break on spaces. */ token = strtok (line_dup, " "); while (token != NULL) @@ -764,19 +862,23 @@ format_and_print_line (const char *prefix, const char *line, /* If however a token is much larger than the maximum line length, all bets are off and we simply print the token on the next line. */ - if (remaining_chars <= strlen (token)) + if (remaining_chars <= (int) strlen (token)) { - printf ("\n%*c", TABULATION, ' '); + if (printf ("\n%*c", TABULATION, ' ') < 0) + return -1; remaining_chars = line_length - TABULATION; } - printf ("%s ", token); + if (printf ("%s ", token) < 0) + return -1; remaining_chars -= strlen (token) + 1; /* account for " " */ token = strtok (NULL, " "); } - printf ("\n"); + if (printf ("\n") < 0) + return -1; xfree (line_dup); + return 0; } static void @@ -789,87 +891,112 @@ print_version (void) char *env_wgetrc, *user_wgetrc; int i; - printf (_("GNU Wget %s built on %s.\n\n"), version_string, OS_TYPE); + if (printf (_("GNU Wget %s built on %s.\n\n"), version_string, OS_TYPE) < 0) + exit (3); for (i = 0; compiled_features[i] != NULL; ) { int line_length = MAX_CHARS_PER_LINE; while ((line_length > 0) && (compiled_features[i] != NULL)) { - printf ("%s ", compiled_features[i]); + if (printf ("%s ", compiled_features[i]) < 0) + exit (3); line_length -= strlen (compiled_features[i]) + 2; i++; } - printf ("\n"); + if (printf ("\n") < 0) + exit (3); } - printf ("\n"); + if (printf ("\n") < 0) + exit (3); /* Handle the case when $WGETRC is unset and $HOME/.wgetrc is absent. */ - printf ("%s\n", wgetrc_title); + if (printf ("%s\n", wgetrc_title) < 0) + exit (3); + env_wgetrc = wgetrc_env_file_name (); if (env_wgetrc && *env_wgetrc) { - printf (_(" %s (env)\n"), env_wgetrc); + if (printf (_(" %s (env)\n"), env_wgetrc) < 0) + exit (3); xfree (env_wgetrc); } user_wgetrc = wgetrc_user_file_name (); if (user_wgetrc) { - printf (_(" %s (user)\n"), user_wgetrc); + if (printf (_(" %s (user)\n"), user_wgetrc) < 0) + exit (3); xfree (user_wgetrc); } #ifdef SYSTEM_WGETRC - printf (_(" %s (system)\n"), SYSTEM_WGETRC); + if (printf (_(" %s (system)\n"), SYSTEM_WGETRC) < 0) + exit (3); #endif #ifdef ENABLE_NLS - format_and_print_line (locale_title, - LOCALEDIR, - MAX_CHARS_PER_LINE); + if (format_and_print_line (locale_title, + LOCALEDIR, + MAX_CHARS_PER_LINE) < 0) + exit (3); #endif /* def ENABLE_NLS */ if (compilation_string != NULL) - format_and_print_line (compile_title, - compilation_string, - MAX_CHARS_PER_LINE); + if (format_and_print_line (compile_title, + compilation_string, + MAX_CHARS_PER_LINE) < 0) + exit (3); if (link_string != NULL) - format_and_print_line (link_title, - link_string, - MAX_CHARS_PER_LINE); + if (format_and_print_line (link_title, + link_string, + MAX_CHARS_PER_LINE) < 0) + exit (3); + + if (printf ("\n") < 0) + exit (3); - printf ("\n"); /* TRANSLATORS: When available, an actual copyright character - (cirle-c) should be used in preference to "(C)". */ - fputs (_("\ -Copyright (C) 2009 Free Software Foundation, Inc.\n"), stdout); - fputs (_("\ + (circle-c) should be used in preference to "(C)". */ + if (printf (_("\ +Copyright (C) %s Free Software Foundation, Inc.\n"), "2014") < 0) + exit (3); + if (fputs (_("\ License GPLv3+: GNU GPL version 3 or later\n\ .\n\ This is free software: you are free to change and redistribute it.\n\ -There is NO WARRANTY, to the extent permitted by law.\n"), stdout); +There is NO WARRANTY, to the extent permitted by law.\n"), stdout) < 0) + exit (3); /* TRANSLATORS: When available, please use the proper diacritics for names such as this one. See en_US.po for reference. */ - fputs (_("\nOriginally written by Hrvoje Niksic .\n"), - stdout); - fputs (_("Please send bug reports and questions to .\n"), - stdout); + if (fputs (_("\nOriginally written by Hrvoje Niksic .\n"), + stdout) < 0) + exit (3); + if (fputs (_("Please send bug reports and questions to .\n"), + stdout) < 0) + exit (3); + exit (0); } -char *program_name; /* Needed by lib/error.c. */ +static char *program_name; /* Needed by lib/error.c. */ +char *program_argstring; /* Needed by wget_warc.c. */ int main (int argc, char **argv) { char **url, **t; int i, ret, longindex; - int nurl, status; + int nurl; bool append_to_log = false; + total_downloaded_bytes = 0; + program_name = argv[0]; + struct ptimer *timer = ptimer_new (); + double start_time = ptimer_measure (timer); + i18n_initialize (); /* Construct the name of the executable, without the directory part. */ @@ -889,16 +1016,84 @@ main (int argc, char **argv) windows_main ((char **) &exec_name); #endif - /* Set option defaults; read the system wgetrc and ~/.wgetrc. */ - initialize (); + /* Construct the arguments string. */ + int argstring_length = 1; + for (i = 1; i < argc; i++) + argstring_length += strlen (argv[i]) + 2 + 1; + char *p = program_argstring = malloc (argstring_length * sizeof (char)); + if (p == NULL) + { + fprintf (stderr, _("Memory allocation problem\n")); + exit (2); + } + for (i = 1; i < argc; i++) + { + *p++ = '"'; + int arglen = strlen (argv[i]); + memcpy (p, argv[i], arglen); + p += arglen; + *p++ = '"'; + *p++ = ' '; + } + *p = '\0'; + + /* Load the hard-coded defaults. */ + defaults (); init_switches (); + + /* This separate getopt_long is needed to find the user config file + option ("--config") and parse it before the other user options. */ + longindex = -1; + int retconf; + bool use_userconfig = false; + bool noconfig = false; + + while ((retconf = getopt_long (argc, argv, + short_options, long_options, &longindex)) != -1) + { + int confval; + struct cmdline_option *config_opt; + + /* There is no short option for "--config". */ + if (longindex >= 0) + { + confval = long_options[longindex].val; + config_opt = &option_data[confval & ~BOOLEAN_NEG_MARKER]; + if (strcmp (config_opt->long_name, "no-config") == 0) + { + noconfig = true; + break; + } + else if (strcmp (config_opt->long_name, "config") == 0) + { + bool userrc_ret = true; + userrc_ret &= run_wgetrc (optarg); + use_userconfig = true; + if (userrc_ret) + break; + else + { + fprintf (stderr, _("Exiting due to error in %s\n"), optarg); + exit (2); + } + } + } + } + + /* If the user did not specify a config, read the system wgetrc and ~/.wgetrc. */ + if (noconfig == false && use_userconfig == false) + initialize (); + + opterr = 0; + optind = 0; + longindex = -1; while ((ret = getopt_long (argc, argv, short_options, long_options, &longindex)) != -1) { int val; - struct cmdline_option *opt; + struct cmdline_option *cmdopt; /* If LONGINDEX is unchanged, it means RET is referring a short option. */ @@ -906,9 +1101,10 @@ main (int argc, char **argv) { if (ret == '?') { - print_usage (0); - printf ("\n"); - printf (_("Try `%s --help' for more options.\n"), exec_name); + print_usage (1); + fprintf (stderr, "\n"); + fprintf (stderr, _("Try `%s --help' for more options.\n"), + exec_name); exit (2); } /* Find the short option character in the mapping. */ @@ -919,31 +1115,31 @@ main (int argc, char **argv) /* Use the retrieved value to locate the option in the option_data array, and to see if we're dealing with the negated "--no-FOO" variant of the boolean option "--foo". */ - opt = &option_data[val & ~BOOLEAN_NEG_MARKER]; - switch (opt->type) + cmdopt = &option_data[val & ~BOOLEAN_NEG_MARKER]; + switch (cmdopt->type) { case OPT_VALUE: - setoptval (opt->data, optarg, opt->long_name); + setoptval (cmdopt->data, optarg, cmdopt->long_name); break; case OPT_BOOLEAN: if (optarg) /* The user has specified a value -- use it. */ - setoptval (opt->data, optarg, opt->long_name); + setoptval (cmdopt->data, optarg, cmdopt->long_name); else { /* NEG is true for `--no-FOO' style boolean options. */ bool neg = !!(val & BOOLEAN_NEG_MARKER); - setoptval (opt->data, neg ? "0" : "1", opt->long_name); + setoptval (cmdopt->data, neg ? "0" : "1", cmdopt->long_name); } break; case OPT_FUNCALL: { - void (*func) (void) = (void (*) (void)) opt->data; + void (*func) (void) = (void (*) (void)) cmdopt->data; func (); } break; case OPT__APPEND_OUTPUT: - setoptval ("logfile", optarg, opt->long_name); + setoptval ("logfile", optarg, cmdopt->long_name); append_to_log = true; break; case OPT__EXECUTE: @@ -954,24 +1150,23 @@ main (int argc, char **argv) /* We support real --no-FOO flags now, but keep these short options for convenience and backward compatibility. */ - char *p; - for (p = optarg; *p; p++) + for (p = optarg; p && *p; p++) switch (*p) { case 'v': - setoptval ("verbose", "0", opt->long_name); + setoptval ("verbose", "0", cmdopt->long_name); break; case 'H': - setoptval ("addhostdir", "0", opt->long_name); + setoptval ("addhostdir", "0", cmdopt->long_name); break; case 'd': - setoptval ("dirstruct", "0", opt->long_name); + setoptval ("dirstruct", "0", cmdopt->long_name); break; case 'c': - setoptval ("noclobber", "1", opt->long_name); + setoptval ("noclobber", "1", cmdopt->long_name); break; case 'p': - setoptval ("noparent", "1", opt->long_name); + setoptval ("noparent", "1", cmdopt->long_name); break; default: fprintf (stderr, _("%s: illegal option -- `-n%c'\n"), @@ -995,12 +1190,12 @@ main (int argc, char **argv) flag = (*optarg == '1' || c_tolower (*optarg) == 'y' || (c_tolower (optarg[0]) == 'o' && c_tolower (optarg[1]) == 'n')); - setoptval (opt->type == OPT__PARENT ? "noparent" : "noclobber", - flag ? "0" : "1", opt->long_name); + setoptval (cmdopt->type == OPT__PARENT ? "noparent" : "noclobber", + flag ? "0" : "1", cmdopt->long_name); break; } case OPT__DONT_REMOVE_LISTING: - setoptval ("removelisting", "0", opt->long_name); + setoptval ("removelisting", "0", cmdopt->long_name); break; } @@ -1009,9 +1204,31 @@ main (int argc, char **argv) nurl = argc - optind; + /* If we do not have Debug support compiled in AND Wget is invoked with the + * --debug switch, instead of failing, we silently turn it into a no-op. For + * this no-op, we explicitly set opt.debug to false and hence none of the + * Debug output messages will be printed. + */ +#ifndef ENABLE_DEBUG + if (opt.debug) + { + fprintf (stderr, _("Debugging support not compiled in. " + "Ignoring --debug flag.\n")); + opt.debug = false; + } +#endif + /* All user options have now been processed, so it's now safe to do interoption dependency checks. */ + if (opt.noclobber && opt.convert_links) + { + fprintf (stderr, + _("Both --no-clobber and --convert-links were specified," + " only --convert-links will be used.\n")); + opt.noclobber = false; + } + if (opt.reclevel == 0) opt.reclevel = INFINITE_RECURSION; /* see recur.h for commentary */ @@ -1031,6 +1248,9 @@ main (int argc, char **argv) if (opt.verbose == -1) opt.verbose = !opt.quiet; + if (opt.verbose == 1) + opt.show_progress = true; + /* Sanity checks. */ if (opt.verbose && opt.quiet) { @@ -1089,6 +1309,48 @@ for details.\n\n")); } } + if (opt.warc_filename != 0) + { + if (opt.noclobber) + { + fprintf (stderr, + _("WARC output does not work with --no-clobber, " + "--no-clobber will be disabled.\n")); + opt.noclobber = false; + } + if (opt.timestamping) + { + fprintf (stderr, + _("WARC output does not work with timestamping, " + "timestamping will be disabled.\n")); + opt.timestamping = false; + } + if (opt.spider) + { + fprintf (stderr, + _("WARC output does not work with --spider.\n")); + exit (1); + } + if (opt.always_rest || opt.start_pos >= 0) + { + fprintf (stderr, + _("WARC output does not work with --continue or" + " --start-pos, they will be disabled.\n")); + opt.always_rest = false; + opt.start_pos = -1; + } + if (opt.warc_cdx_dedup_filename != 0 && !opt.warc_digests_enabled) + { + fprintf (stderr, + _("Digests are disabled; WARC deduplication will " + "not find duplicate records.\n")); + } + if (opt.warc_keep_log) + { + opt.progress_type = xstrdup ("dot"); + } + } + if (opt.ask_passwd && opt.passwd) { fprintf (stderr, @@ -1097,18 +1359,113 @@ for details.\n\n")); exit (1); } + if (opt.start_pos >= 0 && opt.always_rest) + { + fprintf (stderr, + _("Specifying both --start-pos and --continue is not " + "recommended; --continue will be disabled.\n")); + opt.always_rest = false; + } + if (!nurl && !opt.input_filename) { /* No URL specified. */ fprintf (stderr, _("%s: missing URL\n"), exec_name); print_usage (1); - printf ("\n"); + fprintf (stderr, "\n"); /* #### Something nicer should be printed here -- similar to the pre-1.5 `--help' page. */ fprintf (stderr, _("Try `%s --help' for more options.\n"), exec_name); exit (1); } + /* Compile the regular expressions. */ + switch (opt.regex_type) + { +#ifdef HAVE_LIBPCRE + case regex_type_pcre: + opt.regex_compile_fun = compile_pcre_regex; + opt.regex_match_fun = match_pcre_regex; + break; +#endif + + case regex_type_posix: + default: + opt.regex_compile_fun = compile_posix_regex; + opt.regex_match_fun = match_posix_regex; + break; + } + if (opt.acceptregex_s) + { + opt.acceptregex = opt.regex_compile_fun (opt.acceptregex_s); + if (!opt.acceptregex) + exit (1); + } + if (opt.rejectregex_s) + { + opt.rejectregex = opt.regex_compile_fun (opt.rejectregex_s); + if (!opt.rejectregex) + exit (1); + } + if (opt.post_data || opt.post_file_name) + { + if (opt.post_data && opt.post_file_name) + { + fprintf (stderr, _("You cannot specify both --post-data and --post-file.\n")); + exit (1); + } + else if (opt.method) + { + fprintf (stderr, _("You cannot use --post-data or --post-file along with --method. " + "--method expects data through --body-data and --body-file options")); + exit (1); + } + } + if (opt.body_data || opt.body_file) + { + if (!opt.method) + { + fprintf (stderr, _("You must specify a method through --method=HTTPMethod " + "to use with --body-data or --body-file.\n")); + exit (1); + } + else if (opt.body_data && opt.body_file) + { + fprintf (stderr, _("You cannot specify both --body-data and --body-file.\n")); + exit (1); + } + } + + /* Set various options as required for opt.method. */ + + /* When user specifies HEAD as the method, we do not wish to download any + files. Hence, set wget to run in spider mode. */ + if (opt.method && strcasecmp (opt.method, "HEAD") == 0) + setoptval ("spider", "1", "spider"); + + /* Convert post_data to body-data and post_file_name to body-file options. + This is required so as to remove redundant code later on in gethttp(). + The --post-data and --post-file options may also be removed in + the future hence it makes sense to convert them to aliases for + the more generic --method options. + This MUST occur only after the sanity checks so as to prevent the + user from setting both post and body options simultaneously. + */ + if (opt.post_data || opt.post_file_name) + { + setoptval ("method", "POST", "method"); + if (opt.post_data) + { + setoptval ("bodydata", opt.post_data, "body-data"); + opt.post_data = NULL; + } + else + { + setoptval ("bodyfile", opt.post_file_name, "body-file"); + opt.post_file_name = NULL; + } + } + #ifdef ENABLE_IRI if (opt.enable_iri) { @@ -1122,6 +1479,7 @@ for details.\n\n")); opt.encoding_remote = NULL; } #else + memset (&dummy_iri, 0, sizeof (dummy_iri)); if (opt.enable_iri || opt.locale || opt.encoding_remote) { /* sXXXav : be more specific... */ @@ -1149,11 +1507,16 @@ for details.\n\n")); /* Initialize progress. Have to do this after the options are processed so we know where the log file is. */ - if (opt.verbose) + if (opt.show_progress) set_progress_implementation (opt.progress_type); /* Fill in the arguments. */ url = alloca_array (char *, nurl + 1); + if (url == NULL) + { + fprintf (stderr, _("Memory allocation problem\n")); + exit (2); + } for (i = 0; i < nurl; i++, optind++) { char *rewritten = rewrite_shorthand_url (argv[optind]); @@ -1167,6 +1530,10 @@ for details.\n\n")); /* Initialize logging. */ log_init (opt.lfilename, append_to_log); + /* Open WARC file. */ + if (opt.warc_filename != 0) + warc_init (); + DEBUGP (("DEBUG output created by Wget %s on %s.\n\n", version_string, OS_TYPE)); @@ -1185,14 +1552,7 @@ for details.\n\n")); if (HYPHENP (opt.output_document)) { #ifdef WINDOWS - FILE *result; - result = freopen ("CONOUT$", "wb", stdout); - if (result == NULL) - { - logputs (LOG_NOTQUIET, _("\ -WARNING: Can't reopen standard output in binary mode;\n\ - downloaded file may contain inappropriate line endings.\n")); - } + _setmode (_fileno (stdout), _O_BINARY); #endif output_stream = stdout; } @@ -1264,7 +1624,6 @@ outputting to a regular file.\n")); signal (SIGWINCH, progress_handle_sigwinch); #endif - status = RETROK; /* initialize it, just-in-case */ /* Retrieve the URLs from argument list. */ for (t = url; *t; t++) { @@ -1284,7 +1643,7 @@ outputting to a regular file.\n")); char *error = url_error (*t, url_err); logprintf (LOG_NOTQUIET, "%s: %s.\n",*t, error); xfree (error); - status = URLERROR; + inform_exit_status (URLERROR); } else { @@ -1297,17 +1656,17 @@ outputting to a regular file.\n")); if (url_scheme (*t) == SCHEME_FTP) opt.follow_ftp = 1; - status = retrieve_tree (url_parsed, NULL); + retrieve_tree (url_parsed, NULL); opt.follow_ftp = old_follow_ftp; } else { - status = retrieve_url (url_parsed, *t, &filename, &redirected_URL, - NULL, &dt, opt.recursive, iri, true); + retrieve_url (url_parsed, *t, &filename, &redirected_URL, NULL, + &dt, opt.recursive, iri, true); } - if (opt.delete_after && file_exists_p(filename)) + if (opt.delete_after && filename != NULL && file_exists_p (filename)) { DEBUGP (("Removing file due to --delete-after in main():\n")); logprintf (LOG_VERBOSE, _("Removing %s.\n"), filename); @@ -1325,7 +1684,9 @@ outputting to a regular file.\n")); if (opt.input_filename) { int count; + int status; status = retrieve_from_file (opt.input_filename, opt.force_html, &count); + inform_exit_status (status); if (!count) logprintf (LOG_NOTQUIET, _("No URLs found in %s.\n"), opt.input_filename); @@ -1342,13 +1703,23 @@ outputting to a regular file.\n")); && total_downloaded_bytes != 0) { + double end_time = ptimer_measure (timer); + ptimer_destroy (timer); + + char *wall_time = xstrdup (secs_to_human_time (end_time - start_time)); + char *download_time = xstrdup (secs_to_human_time (total_download_time)); logprintf (LOG_NOTQUIET, - _("FINISHED --%s--\nDownloaded: %d files, %s in %s (%s)\n"), - datetime_str (time (NULL)), - numurls, - human_readable (total_downloaded_bytes), - secs_to_human_time (total_download_time), - retr_rate (total_downloaded_bytes, total_download_time)); + _("FINISHED --%s--\nTotal wall clock time: %s\n" + "Downloaded: %d files, %s in %s (%s)\n"), + datetime_str (time (NULL)), + wall_time, + numurls, + human_readable (total_downloaded_bytes), + download_time, + retr_rate (total_downloaded_bytes, total_download_time)); + xfree (wall_time); + xfree (download_time); + /* Print quota warning, if exceeded. */ if (opt.quota && total_downloaded_bytes > opt.quota) logprintf (LOG_NOTQUIET, @@ -1362,12 +1733,9 @@ outputting to a regular file.\n")); if (opt.convert_links && !opt.delete_after) convert_all_links (); - log_close (); - for (i = 0; i < nurl; i++) - xfree (url[i]); cleanup (); - return get_exit_status (); + exit (get_exit_status ()); } #endif /* TESTING */