X-Git-Url: http://sjero.net/git/?p=wget;a=blobdiff_plain;f=src%2Fmain.c;h=d4fea4f3619e81fc11e4291b4689fed39cdb729e;hp=ed3349147825ac06415d7a4bc4c29785a927bfac;hb=38a7829dcb4eb5dba28dbf0f05c6a80fea9217f8;hpb=3d5863424bd4a4354311c449e9d318057996d16e
diff --git a/src/main.c b/src/main.c
index ed334914..29c13242 100644
--- a/src/main.c
+++ b/src/main.c
@@ -1,11 +1,13 @@
/* Command line parsing.
- Copyright (C) 1996-2005 Free Software Foundation, Inc.
+ Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
+ 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 Free Software Foundation,
+ Inc.
This file is part of GNU Wget.
GNU Wget is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
+the Free Software Foundation; either version 3 of the License, or
(at your option) any later version.
GNU Wget is distributed in the hope that it will be useful,
@@ -14,62 +16,90 @@ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
-along with Wget; if not, write to the Free Software Foundation, Inc.,
-51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+along with Wget. If not, see .
-In addition, as a special exception, the Free Software Foundation
-gives permission to link the code of its release of Wget with the
-OpenSSL project's "OpenSSL" library (or with modified versions of it
-that use the same license as the "OpenSSL" library), and distribute
-the linked executables. You must obey the GNU General Public License
-in all respects for all of the code used other than "OpenSSL". If you
-modify this file, you may extend this exception to your version of the
-file, but you are not obligated to do so. If you do not wish to do
-so, delete this exception statement from your version. */
+Additional permission under GNU GPL version 3 section 7
-#include
+If you modify this program, or any covered work, by linking or
+combining it with the OpenSSL project's OpenSSL library (or a
+modified version of that library), containing parts covered by the
+terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
+grants you additional permission to convey the resulting work.
+Corresponding Source for a non-source form of such a combination
+shall include the source code for the parts of OpenSSL used as well
+as that of the covered work. */
+
+#include "wget.h"
#include
#include
-#ifdef HAVE_UNISTD_H
-# include
-#endif /* HAVE_UNISTD_H */
+#include
#include
#include
-#ifdef HAVE_NLS
+#ifdef ENABLE_NLS
# include
#endif
#include
#include
+#include
-#include "wget.h"
+#include "exits.h"
#include "utils.h"
#include "init.h"
#include "retr.h"
#include "recur.h"
#include "host.h"
#include "url.h"
-#include "progress.h" /* for progress_handle_sigwinch */
+#include "progress.h" /* for progress_handle_sigwinch */
#include "convert.h"
-#include "http.h" /* for save_cookies */
+#include "spider.h"
+#include "http.h" /* for save_cookies */
+#include "ptimer.h"
+#include "warc.h"
+#include
+#include
+#include
+
+#ifdef WINDOWS
+# include
+# include
+#endif
-/* On GNU system this will include system-wide getopt.h. */
-#include "getopt.h"
+#ifdef __VMS
+# include "vms.h"
+#endif /* __VMS */
#ifndef PATH_SEPARATOR
# define PATH_SEPARATOR '/'
#endif
+#ifndef ENABLE_IRI
+struct iri dummy_iri;
+#endif
+
struct options opt;
+/* defined in version.c */
extern char *version_string;
+extern char *compilation_string;
+extern char *system_getrc;
+extern char *link_string;
+/* defined in build_info.c */
+extern const char *compiled_features[];
+/* Used for --version output in print_version */
+#define MAX_CHARS_PER_LINE 72
+#define TABULATION 4
#if defined(SIGHUP) || defined(SIGUSR1)
static void redirect_output_signal (int);
#endif
const char *exec_name;
+
+/* Number of successfully downloaded URLs */
+int numurls = 0;
+#ifndef TESTING
/* Initialize I18N/L10N. That amounts to invoking setlocale, and
setting up gettext's message catalog using bindtextdomain and
textdomain. Does nothing if NLS is disabled or missing. */
@@ -77,14 +107,14 @@ const char *exec_name;
static void
i18n_initialize (void)
{
- /* HAVE_NLS implies existence of functions invoked here. */
-#ifdef HAVE_NLS
+ /* ENABLE_NLS implies existence of functions invoked here. */
+#ifdef ENABLE_NLS
/* Set the current locale. */
setlocale (LC_ALL, "");
/* Set the text message domain. */
bindtextdomain ("wget", LOCALEDIR);
textdomain ("wget");
-#endif /* HAVE_NLS */
+#endif /* ENABLE_NLS */
}
/* Definition of command-line options. */
@@ -98,12 +128,6 @@ static void print_version (void);
# define IF_SSL(x) NULL
#endif
-#ifdef ENABLE_DEBUG
-# define WHEN_DEBUG(x) x
-#else
-# define WHEN_DEBUG(x) NULL
-#endif
-
struct cmdline_option {
const char *long_name;
char short_name;
@@ -120,19 +144,25 @@ struct cmdline_option {
OPT__NO,
OPT__PARENT
} type;
- const void *data; /* for standard options */
- int argtype; /* for non-standard options */
+ const void *data; /* for standard options */
+ int argtype; /* for non-standard options */
};
static struct cmdline_option option_data[] =
{
{ "accept", 'A', OPT_VALUE, "accept", -1 },
+ { "accept-regex", 0, OPT_VALUE, "acceptregex", -1 },
+ { "adjust-extension", 'E', OPT_BOOLEAN, "adjustextension", -1 },
{ "append-output", 'a', OPT__APPEND_OUTPUT, NULL, required_argument },
+ { "ask-password", 0, OPT_BOOLEAN, "askpassword", -1 },
+ { "auth-no-challenge", 0, OPT_BOOLEAN, "authnochallenge", -1 },
{ "background", 'b', OPT_BOOLEAN, "background", -1 },
{ "backup-converted", 'K', OPT_BOOLEAN, "backupconverted", -1 },
{ "backups", 0, OPT_BOOLEAN, "backups", -1 },
{ "base", 'B', OPT_VALUE, "base", -1 },
{ "bind-address", 0, OPT_VALUE, "bindaddress", -1 },
+ { "body-data", 0, OPT_VALUE, "bodydata", -1 },
+ { "body-file", 0, OPT_VALUE, "bodyfile", -1 },
{ IF_SSL ("ca-certificate"), 0, OPT_VALUE, "cacertificate", -1 },
{ IF_SSL ("ca-directory"), 0, OPT_VALUE, "cadirectory", -1 },
{ "cache", 0, OPT_BOOLEAN, "cache", -1 },
@@ -140,12 +170,16 @@ static struct cmdline_option option_data[] =
{ IF_SSL ("certificate-type"), 0, OPT_VALUE, "certificatetype", -1 },
{ IF_SSL ("check-certificate"), 0, OPT_BOOLEAN, "checkcertificate", -1 },
{ "clobber", 0, OPT__CLOBBER, NULL, optional_argument },
+ { "config", 0, OPT_VALUE, "chooseconfig", -1 },
{ "connect-timeout", 0, OPT_VALUE, "connecttimeout", -1 },
{ "continue", 'c', OPT_BOOLEAN, "continue", -1 },
{ "convert-links", 'k', OPT_BOOLEAN, "convertlinks", -1 },
+ { "content-disposition", 0, OPT_BOOLEAN, "contentdisposition", -1 },
+ { "content-on-error", 0, OPT_BOOLEAN, "contentonerror", -1 },
{ "cookies", 0, OPT_BOOLEAN, "cookies", -1 },
{ "cut-dirs", 0, OPT_VALUE, "cutdirs", -1 },
- { WHEN_DEBUG ("debug"), 'd', OPT_BOOLEAN, "debug", -1 },
+ { "debug", 'd', OPT_BOOLEAN, "debug", -1 },
+ { "default-page", 0, OPT_VALUE, "defaultpage", -1 },
{ "delete-after", 0, OPT_BOOLEAN, "deleteafter", -1 },
{ "directories", 0, OPT_BOOLEAN, "dirstruct", -1 },
{ "directory-prefix", 'P', OPT_VALUE, "dirprefix", -1 },
@@ -153,7 +187,7 @@ static struct cmdline_option option_data[] =
{ "dns-timeout", 0, OPT_VALUE, "dnstimeout", -1 },
{ "domains", 'D', OPT_VALUE, "domains", -1 },
{ "dont-remove-listing", 0, OPT__DONT_REMOVE_LISTING, NULL, no_argument },
- { "dot-style", 0, OPT_VALUE, "dotstyle", -1 },
+ { "dot-style", 0, OPT_VALUE, "dotstyle", -1 }, /* deprecated */
{ "egd-file", 0, OPT_VALUE, "egdfile", -1 },
{ "exclude-directories", 'X', OPT_VALUE, "excludedirectories", -1 },
{ "exclude-domains", 0, OPT_VALUE, "excludedomains", -1 },
@@ -163,17 +197,22 @@ static struct cmdline_option option_data[] =
{ "force-directories", 'x', OPT_BOOLEAN, "dirstruct", -1 },
{ "force-html", 'F', OPT_BOOLEAN, "forcehtml", -1 },
{ "ftp-password", 0, OPT_VALUE, "ftppassword", -1 },
+#ifdef __VMS
+ { "ftp-stmlf", 0, OPT_BOOLEAN, "ftpstmlf", -1 },
+#endif /* def __VMS */
{ "ftp-user", 0, OPT_VALUE, "ftpuser", -1 },
{ "glob", 0, OPT_BOOLEAN, "glob", -1 },
{ "header", 0, OPT_VALUE, "header", -1 },
{ "help", 'h', OPT_FUNCALL, (void *)print_help, no_argument },
{ "host-directories", 0, OPT_BOOLEAN, "addhostdir", -1 },
- { "html-extension", 'E', OPT_BOOLEAN, "htmlextension", -1 },
+ { "html-extension", 'E', OPT_BOOLEAN, "adjustextension", -1 }, /* deprecated */
{ "htmlify", 0, OPT_BOOLEAN, "htmlify", -1 },
{ "http-keep-alive", 0, OPT_BOOLEAN, "httpkeepalive", -1 },
{ "http-passwd", 0, OPT_VALUE, "httppassword", -1 }, /* deprecated */
{ "http-password", 0, OPT_VALUE, "httppassword", -1 },
{ "http-user", 0, OPT_VALUE, "httpuser", -1 },
+ { IF_SSL ("https-only"), 0, OPT_BOOLEAN, "httpsonly", -1 },
+ { "ignore-case", 0, OPT_BOOLEAN, "ignorecase", -1 },
{ "ignore-length", 0, OPT_BOOLEAN, "ignorelength", -1 },
{ "ignore-tags", 0, OPT_VALUE, "ignoretags", -1 },
{ "include-directories", 'I', OPT_VALUE, "includedirectories", -1 },
@@ -182,13 +221,18 @@ static struct cmdline_option option_data[] =
{ "inet6-only", '6', OPT_BOOLEAN, "inet6only", -1 },
#endif
{ "input-file", 'i', OPT_VALUE, "input", -1 },
+ { "iri", 0, OPT_BOOLEAN, "iri", -1 },
{ "keep-session-cookies", 0, OPT_BOOLEAN, "keepsessioncookies", -1 },
{ "level", 'l', OPT_VALUE, "reclevel", -1 },
{ "limit-rate", 0, OPT_VALUE, "limitrate", -1 },
{ "load-cookies", 0, OPT_VALUE, "loadcookies", -1 },
+ { "local-encoding", 0, OPT_VALUE, "localencoding", -1 },
+ { "max-redirect", 0, OPT_VALUE, "maxredirect", -1 },
+ { "method", 0, OPT_VALUE, "method", -1 },
{ "mirror", 'm', OPT_BOOLEAN, "mirror", -1 },
{ "no", 'n', OPT__NO, NULL, required_argument },
{ "no-clobber", 0, OPT_BOOLEAN, "noclobber", -1 },
+ { "no-config", 0, OPT_BOOLEAN, "noconfig", -1},
{ "no-parent", 0, OPT_BOOLEAN, "noparent", -1 },
{ "output-document", 'O', OPT_VALUE, "outputdocument", -1 },
{ "output-file", 'o', OPT_VALUE, "logfile", -1 },
@@ -203,6 +247,7 @@ static struct cmdline_option option_data[] =
{ IF_SSL ("private-key"), 0, OPT_VALUE, "privatekey", -1 },
{ IF_SSL ("private-key-type"), 0, OPT_VALUE, "privatekeytype", -1 },
{ "progress", 0, OPT_VALUE, "progress", -1 },
+ { "show-progress", 0, OPT_BOOLEAN, "showprogress", -1 },
{ "protocol-directories", 0, OPT_BOOLEAN, "protocoldirectories", -1 },
{ "proxy", 0, OPT_BOOLEAN, "useproxy", -1 },
{ "proxy__compat", 'Y', OPT_VALUE, "useproxy", -1 }, /* back-compatible */
@@ -216,9 +261,13 @@ static struct cmdline_option option_data[] =
{ "read-timeout", 0, OPT_VALUE, "readtimeout", -1 },
{ "recursive", 'r', OPT_BOOLEAN, "recursive", -1 },
{ "referer", 0, OPT_VALUE, "referer", -1 },
+ { "regex-type", 0, OPT_VALUE, "regextype", -1 },
{ "reject", 'R', OPT_VALUE, "reject", -1 },
+ { "reject-regex", 0, OPT_VALUE, "rejectregex", -1 },
{ "relative", 'L', OPT_BOOLEAN, "relativeonly", -1 },
+ { "remote-encoding", 0, OPT_VALUE, "remoteencoding", -1 },
{ "remove-listing", 0, OPT_BOOLEAN, "removelisting", -1 },
+ { "report-speed", 0, OPT_BOOLEAN, "reportspeed", -1 },
{ "restrict-file-names", 0, OPT_BOOLEAN, "restrictfilenames", -1 },
{ "retr-symlinks", 0, OPT_BOOLEAN, "retrsymlinks", -1 },
{ "retry-connrefused", 0, OPT_BOOLEAN, "retryconnrefused", -1 },
@@ -228,10 +277,14 @@ static struct cmdline_option option_data[] =
{ "server-response", 'S', OPT_BOOLEAN, "serverresponse", -1 },
{ "span-hosts", 'H', OPT_BOOLEAN, "spanhosts", -1 },
{ "spider", 0, OPT_BOOLEAN, "spider", -1 },
+ { "start-pos", 0, OPT_VALUE, "startpos", -1 },
{ "strict-comments", 0, OPT_BOOLEAN, "strictcomments", -1 },
{ "timeout", 'T', OPT_VALUE, "timeout", -1 },
{ "timestamping", 'N', OPT_BOOLEAN, "timestamping", -1 },
{ "tries", 't', OPT_VALUE, "tries", -1 },
+ { "unlink", 0, OPT_BOOLEAN, "unlink", -1 },
+ { "trust-server-names", 0, OPT_BOOLEAN, "trustservernames", -1 },
+ { "use-server-timestamps", 0, OPT_BOOLEAN, "useservertimestamps", -1 },
{ "user", 0, OPT_VALUE, "user", -1 },
{ "user-agent", 'U', OPT_VALUE, "useragent", -1 },
{ "verbose", 'v', OPT_BOOLEAN, "verbose", -1 },
@@ -239,9 +292,22 @@ static struct cmdline_option option_data[] =
{ "version", 'V', OPT_FUNCALL, (void *) print_version, no_argument },
{ "wait", 'w', OPT_VALUE, "wait", -1 },
{ "waitretry", 0, OPT_VALUE, "waitretry", -1 },
+ { "warc-cdx", 0, OPT_BOOLEAN, "warccdx", -1 },
+#ifdef HAVE_LIBZ
+ { "warc-compression", 0, OPT_BOOLEAN, "warccompression", -1 },
+#endif
+ { "warc-dedup", 0, OPT_VALUE, "warccdxdedup", -1 },
+ { "warc-digests", 0, OPT_BOOLEAN, "warcdigests", -1 },
+ { "warc-file", 0, OPT_VALUE, "warcfile", -1 },
+ { "warc-header", 0, OPT_VALUE, "warcheader", -1 },
+ { "warc-keep-log", 0, OPT_BOOLEAN, "warckeeplog", -1 },
+ { "warc-max-size", 0, OPT_VALUE, "warcmaxsize", -1 },
+ { "warc-tempdir", 0, OPT_VALUE, "warctempdir", -1 },
+#ifdef USE_WATT32
+ { "wdebug", 0, OPT_BOOLEAN, "wdebug", -1 },
+#endif
};
-#undef WHEN_DEBUG
#undef IF_SSL
/* Return a string that contains S with "no-" prepended. The string
@@ -251,7 +317,7 @@ static struct cmdline_option option_data[] =
static char *
no_prefix (const char *s)
{
- static char buffer[1024];
+ static char buffer[2048];
static char *p = buffer;
char *cp = p;
@@ -282,58 +348,58 @@ static void
init_switches (void)
{
char *p = short_options;
- int i, o = 0;
+ size_t i, o = 0;
for (i = 0; i < countof (option_data); i++)
{
- struct cmdline_option *opt = &option_data[i];
+ struct cmdline_option *cmdopt = &option_data[i];
struct option *longopt;
- if (!opt->long_name)
- /* The option is disabled. */
- continue;
+ if (!cmdopt->long_name)
+ /* The option is disabled. */
+ continue;
longopt = &long_options[o++];
- longopt->name = opt->long_name;
+ longopt->name = cmdopt->long_name;
longopt->val = i;
- if (opt->short_name)
- {
- *p++ = opt->short_name;
- optmap[opt->short_name - 32] = longopt - long_options;
- }
- switch (opt->type)
- {
- case OPT_VALUE:
- longopt->has_arg = required_argument;
- if (opt->short_name)
- *p++ = ':';
- break;
- case OPT_BOOLEAN:
- /* Specify an optional argument for long options, so that
- --option=off works the same as --no-option, for
- compatibility with pre-1.10 Wget. However, don't specify
- optional arguments short-option booleans because they
- prevent combining of short options. */
- longopt->has_arg = optional_argument;
- /* For Boolean options, add the "--no-FOO" variant, which is
- identical to "--foo", except it has opposite meaning and
- it doesn't allow an argument. */
- longopt = &long_options[o++];
- longopt->name = no_prefix (opt->long_name);
- longopt->has_arg = no_argument;
- /* Mask the value so we'll be able to recognize that we're
- dealing with the false value. */
- longopt->val = i | BOOLEAN_NEG_MARKER;
- break;
- default:
- assert (opt->argtype != -1);
- longopt->has_arg = opt->argtype;
- if (opt->short_name)
- {
- if (longopt->has_arg == required_argument)
- *p++ = ':';
- /* Don't handle optional_argument */
- }
- }
+ if (cmdopt->short_name)
+ {
+ *p++ = cmdopt->short_name;
+ optmap[cmdopt->short_name - 32] = longopt - long_options;
+ }
+ switch (cmdopt->type)
+ {
+ case OPT_VALUE:
+ longopt->has_arg = required_argument;
+ if (cmdopt->short_name)
+ *p++ = ':';
+ break;
+ case OPT_BOOLEAN:
+ /* Specify an optional argument for long options, so that
+ --option=off works the same as --no-option, for
+ compatibility with pre-1.10 Wget. However, don't specify
+ optional arguments short-option booleans because they
+ prevent combining of short options. */
+ longopt->has_arg = optional_argument;
+ /* For Boolean options, add the "--no-FOO" variant, which is
+ identical to "--foo", except it has opposite meaning and
+ it doesn't allow an argument. */
+ longopt = &long_options[o++];
+ longopt->name = no_prefix (cmdopt->long_name);
+ longopt->has_arg = no_argument;
+ /* Mask the value so we'll be able to recognize that we're
+ dealing with the false value. */
+ longopt->val = i | BOOLEAN_NEG_MARKER;
+ break;
+ default:
+ assert (cmdopt->argtype != -1);
+ longopt->has_arg = cmdopt->argtype;
+ if (cmdopt->short_name)
+ {
+ if (longopt->has_arg == required_argument)
+ *p++ = ':';
+ /* Don't handle optional_argument */
+ }
+ }
}
/* Terminate short_options. */
*p = '\0';
@@ -343,10 +409,11 @@ init_switches (void)
}
/* Print the usage message. */
-static void
-print_usage (void)
+static int
+print_usage (int error)
{
- printf (_("Usage: %s [OPTION]... [URL]...\n"), exec_name);
+ return fprintf (error ? stderr : stdout,
+ _("Usage: %s [OPTION]... [URL]...\n"), exec_name);
}
/* Print the help message, describing all the available options. If
@@ -381,6 +448,10 @@ Logging and input file:\n"),
#ifdef ENABLE_DEBUG
N_("\
-d, --debug print lots of debugging information.\n"),
+#endif
+#ifdef USE_WATT32
+ N_("\
+ --wdebug print Watt-32 debug output.\n"),
#endif
N_("\
-q, --quiet quiet (no output).\n"),
@@ -389,11 +460,18 @@ Logging and input file:\n"),
N_("\
-nv, --no-verbose turn off verboseness, without being quiet.\n"),
N_("\
- -i, --input-file=FILE download URLs found in FILE.\n"),
+ --report-speed=TYPE Output bandwidth as TYPE. TYPE can be bits.\n"),
+ N_("\
+ -i, --input-file=FILE download URLs found in local or external FILE.\n"),
N_("\
-F, --force-html treat input file as HTML.\n"),
N_("\
- -B, --base=URL prepends URL to relative links in -F -i file.\n"),
+ -B, --base=URL resolves HTML input-file links (-i -F)\n\
+ relative to URL.\n"),
+ N_("\
+ --config=FILE Specify config file to use.\n"),
+ N_("\
+ --no-config Do not read any config file.\n"),
"\n",
N_("\
@@ -406,15 +484,22 @@ Download:\n"),
-O, --output-document=FILE write documents to FILE.\n"),
N_("\
-nc, --no-clobber skip downloads that would download to\n\
- existing files.\n"),
+ existing files (overwriting them).\n"),
N_("\
-c, --continue resume getting a partially-downloaded file.\n"),
+ N_("\
+ --start-pos=OFFSET start downloading from zero-based position OFFSET.\n"),
N_("\
--progress=TYPE select progress gauge type.\n"),
+ N_("\
+ --show-progress display the progress bar in any verbosity mode.\n"),
N_("\
-N, --timestamping don't re-retrieve files unless newer than\n\
local.\n"),
N_("\
+ --no-use-server-timestamps don't set the local file's timestamp by\n\
+ the one on the server.\n"),
+ N_("\
-S, --server-response print server response.\n"),
N_("\
--spider don't download anything.\n"),
@@ -431,9 +516,7 @@ Download:\n"),
N_("\
--waitretry=SECONDS wait 1..SECONDS between retries of a retrieval.\n"),
N_("\
- --random-wait wait from 0...2*WAIT secs between retrievals.\n"),
- N_("\
- -Y, --proxy explicitly turn on proxy.\n"),
+ --random-wait wait from 0.5*WAIT...1.5*WAIT secs between retrievals.\n"),
N_("\
--no-proxy explicitly turn off proxy.\n"),
N_("\
@@ -446,6 +529,8 @@ Download:\n"),
--no-dns-cache disable caching DNS lookups.\n"),
N_("\
--restrict-file-names=OS restrict chars in file names to ones OS allows.\n"),
+ N_("\
+ --ignore-case ignore case when matching files/directories.\n"),
#ifdef ENABLE_IPV6
N_("\
-4, --inet4-only connect only to IPv4 addresses.\n"),
@@ -459,6 +544,16 @@ Download:\n"),
--user=USER set both ftp and http user to USER.\n"),
N_("\
--password=PASS set both ftp and http password to PASS.\n"),
+ N_("\
+ --ask-password prompt for passwords.\n"),
+ N_("\
+ --no-iri turn off IRI support.\n"),
+ N_("\
+ --local-encoding=ENC use ENC as the local encoding for IRIs.\n"),
+ N_("\
+ --remote-encoding=ENC use ENC as the default remote encoding.\n"),
+ N_("\
+ --unlink remove file before clobber.\n"),
"\n",
N_("\
@@ -485,12 +580,17 @@ HTTP options:\n"),
--http-password=PASS set http password to PASS.\n"),
N_("\
--no-cache disallow server-cached data.\n"),
+ N_ ("\
+ --default-page=NAME Change the default page name (normally\n\
+ this is `index.html'.).\n"),
N_("\
- -E, --html-extension save HTML documents with `.html' extension.\n"),
+ -E, --adjust-extension save HTML/CSS documents with proper extensions.\n"),
N_("\
--ignore-length ignore `Content-Length' header field.\n"),
N_("\
--header=STRING insert STRING among the headers.\n"),
+ N_("\
+ --max-redirect maximum redirections allowed per page.\n"),
N_("\
--proxy-user=USER set USER as proxy username.\n"),
N_("\
@@ -515,6 +615,21 @@ HTTP options:\n"),
--post-data=STRING use the POST method; send STRING as the data.\n"),
N_("\
--post-file=FILE use the POST method; send contents of FILE.\n"),
+ N_("\
+ --method=HTTPMethod use method \"HTTPMethod\" in the header.\n"),
+ N_("\
+ --body-data=STRING Send STRING as data. --method MUST be set.\n"),
+ N_("\
+ --body-file=FILE Send contents of FILE. --method MUST be set.\n"),
+ N_("\
+ --content-disposition honor the Content-Disposition header when\n\
+ choosing local file names (EXPERIMENTAL).\n"),
+ N_("\
+ --content-on-error output the received content on server errors.\n"),
+ N_("\
+ --auth-no-challenge send Basic HTTP authentication information\n\
+ without first waiting for the server's\n\
+ challenge.\n"),
"\n",
#ifdef HAVE_SSL
@@ -522,7 +637,9 @@ HTTP options:\n"),
HTTPS (SSL/TLS) options:\n"),
N_("\
--secure-protocol=PR choose secure protocol, one of auto, SSLv2,\n\
- SSLv3, and TLSv1.\n"),
+ SSLv3, TLSv1 and PFS.\n"),
+ N_("\
+ --https-only only follow secure HTTPS links\n"),
N_("\
--no-check-certificate don't validate the server's certificate.\n"),
N_("\
@@ -546,6 +663,10 @@ HTTPS (SSL/TLS) options:\n"),
N_("\
FTP options:\n"),
+#ifdef __VMS
+ N_("\
+ --ftp-stmlf Use Stream_LF format for all binary FTP files.\n"),
+#endif /* def __VMS */
N_("\
--ftp-user=USER set ftp user to USER.\n"),
N_("\
@@ -556,10 +677,35 @@ FTP options:\n"),
--no-glob turn off FTP file name globbing.\n"),
N_("\
--no-passive-ftp disable the \"passive\" transfer mode.\n"),
+ N_("\
+ --preserve-permissions preserve remote file permissions.\n"),
N_("\
--retr-symlinks when recursing, get linked-to files (not dir).\n"),
+ "\n",
+
N_("\
- --preserve-permissions preserve remote file permissions.\n"),
+WARC options:\n"),
+ N_("\
+ --warc-file=FILENAME save request/response data to a .warc.gz file.\n"),
+ N_("\
+ --warc-header=STRING insert STRING into the warcinfo record.\n"),
+ N_("\
+ --warc-max-size=NUMBER set maximum size of WARC files to NUMBER.\n"),
+ N_("\
+ --warc-cdx write CDX index files.\n"),
+ N_("\
+ --warc-dedup=FILENAME do not store records listed in this CDX file.\n"),
+#ifdef HAVE_LIBZ
+ N_("\
+ --no-warc-compression do not compress WARC files with GZIP.\n"),
+#endif
+ N_("\
+ --no-warc-digests do not calculate SHA1 digests.\n"),
+ N_("\
+ --no-warc-keep-log do not store the log file in a WARC record.\n"),
+ N_("\
+ --warc-tempdir=DIRECTORY location for temporary files created by the\n\
+ WARC writer.\n"),
"\n",
N_("\
@@ -571,9 +717,18 @@ Recursive download:\n"),
N_("\
--delete-after delete files locally after downloading them.\n"),
N_("\
- -k, --convert-links make links in downloaded HTML point to local files.\n"),
+ -k, --convert-links make links in downloaded HTML or CSS point to\n\
+ local files.\n"),
+ N_("\
+ --backups=N before writing file X, rotate up to N backup files.\n"),
+
+#ifdef __VMS
+ N_("\
+ -K, --backup-converted before converting file X, back up as X_orig.\n"),
+#else /* def __VMS */
N_("\
-K, --backup-converted before converting file X, back up as X.orig.\n"),
+#endif /* def __VMS [else] */
N_("\
-m, --mirror shortcut for -N -r -l inf --no-remove-listing.\n"),
N_("\
@@ -588,6 +743,17 @@ Recursive accept/reject:\n"),
-A, --accept=LIST comma-separated list of accepted extensions.\n"),
N_("\
-R, --reject=LIST comma-separated list of rejected extensions.\n"),
+ N_("\
+ --accept-regex=REGEX regex matching accepted URLs.\n"),
+ N_("\
+ --reject-regex=REGEX regex matching rejected URLs.\n"),
+#ifdef HAVE_LIBPCRE
+ N_("\
+ --regex-type=TYPE regex type (posix|pcre).\n"),
+#else
+ N_("\
+ --regex-type=TYPE regex type (posix).\n"),
+#endif
N_("\
-D, --domains=LIST comma-separated list of accepted domains.\n"),
N_("\
@@ -605,22 +771,27 @@ Recursive accept/reject:\n"),
N_("\
-I, --include-directories=LIST list of allowed directories.\n"),
N_("\
+ --trust-server-names use the name specified by the redirection\n\
+ url last component.\n"),
+ N_("\
-X, --exclude-directories=LIST list of excluded directories.\n"),
N_("\
-np, --no-parent don't ascend to the parent directory.\n"),
"\n",
-
N_("Mail bug reports and suggestions to .\n")
};
- int i;
+ size_t i;
- printf (_("GNU Wget %s, a non-interactive network retriever.\n"),
- version_string);
- print_usage ();
+ if (printf (_("GNU Wget %s, a non-interactive network retriever.\n"),
+ version_string) < 0)
+ exit (3);
+ if (print_usage (0) < 0)
+ exit (3);
for (i = 0; i < countof (help); i++)
- fputs (_(help[i]), stdout);
+ if (fputs (_(help[i]), stdout) < 0)
+ exit (3);
exit (0);
}
@@ -651,261 +822,784 @@ secs_to_human_time (double interval)
return buf;
}
+static char *
+prompt_for_password (void)
+{
+ if (opt.user)
+ fprintf (stderr, _("Password for user %s: "), quote (opt.user));
+ else
+ fprintf (stderr, _("Password: "));
+ return getpass("");
+}
+
+/* Function that prints the line argument while limiting it
+ to at most line_length. prefix is printed on the first line
+ and an appropriate number of spaces are added on subsequent
+ lines.*/
+static int
+format_and_print_line (const char *prefix, const char *line,
+ int line_length)
+{
+ int remaining_chars;
+ char *line_dup, *token;
+
+ assert (prefix != NULL);
+ assert (line != NULL);
+ assert (line_length > TABULATION);
+
+ line_dup = xstrdup (line);
+
+ if (printf ("%s", prefix) < 0)
+ return -1;
+
+ /* Wrap to new line after prefix. */
+ remaining_chars = 0;
+
+ /* We break on spaces. */
+ token = strtok (line_dup, " ");
+ while (token != NULL)
+ {
+ /* If however a token is much larger than the maximum
+ line length, all bets are off and we simply print the
+ token on the next line. */
+ if (remaining_chars <= (int) strlen (token))
+ {
+ if (printf ("\n%*c", TABULATION, ' ') < 0)
+ return -1;
+ remaining_chars = line_length - TABULATION;
+ }
+ if (printf ("%s ", token) < 0)
+ return -1;
+ remaining_chars -= strlen (token) + 1; /* account for " " */
+ token = strtok (NULL, " ");
+ }
+
+ if (printf ("\n") < 0)
+ return -1;
+
+ xfree (line_dup);
+ return 0;
+}
+
static void
print_version (void)
{
- printf ("GNU Wget %s\n\n", version_string);
- fputs (_("\
-Copyright (C) 2005 Free Software Foundation, Inc.\n"), stdout);
- fputs (_("\
-This program is distributed in the hope that it will be useful,\n\
-but WITHOUT ANY WARRANTY; without even the implied warranty of\n\
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n\
-GNU General Public License for more details.\n"), stdout);
- fputs (_("\nOriginally written by Hrvoje Niksic .\n"),
- stdout);
+ const char *wgetrc_title = _("Wgetrc: ");
+ const char *locale_title = _("Locale: ");
+ const char *compile_title = _("Compile: ");
+ const char *link_title = _("Link: ");
+ char *env_wgetrc, *user_wgetrc;
+ int i;
+
+ if (printf (_("GNU Wget %s built on %s.\n\n"), version_string, OS_TYPE) < 0)
+ exit (3);
+
+ for (i = 0; compiled_features[i] != NULL; )
+ {
+ int line_length = MAX_CHARS_PER_LINE;
+ while ((line_length > 0) && (compiled_features[i] != NULL))
+ {
+ if (printf ("%s ", compiled_features[i]) < 0)
+ exit (3);
+ line_length -= strlen (compiled_features[i]) + 2;
+ i++;
+ }
+ if (printf ("\n") < 0)
+ exit (3);
+ }
+ if (printf ("\n") < 0)
+ exit (3);
+
+ /* Handle the case when $WGETRC is unset and $HOME/.wgetrc is
+ absent. */
+ if (printf ("%s\n", wgetrc_title) < 0)
+ exit (3);
+
+ env_wgetrc = wgetrc_env_file_name ();
+ if (env_wgetrc && *env_wgetrc)
+ {
+ if (printf (_(" %s (env)\n"), env_wgetrc) < 0)
+ exit (3);
+ xfree (env_wgetrc);
+ }
+ user_wgetrc = wgetrc_user_file_name ();
+ if (user_wgetrc)
+ {
+ if (printf (_(" %s (user)\n"), user_wgetrc) < 0)
+ exit (3);
+ xfree (user_wgetrc);
+ }
+#ifdef SYSTEM_WGETRC
+ if (printf (_(" %s (system)\n"), SYSTEM_WGETRC) < 0)
+ exit (3);
+#endif
+
+#ifdef ENABLE_NLS
+ if (format_and_print_line (locale_title,
+ LOCALEDIR,
+ MAX_CHARS_PER_LINE) < 0)
+ exit (3);
+#endif /* def ENABLE_NLS */
+
+ if (compilation_string != NULL)
+ if (format_and_print_line (compile_title,
+ compilation_string,
+ MAX_CHARS_PER_LINE) < 0)
+ exit (3);
+
+ if (link_string != NULL)
+ if (format_and_print_line (link_title,
+ link_string,
+ MAX_CHARS_PER_LINE) < 0)
+ exit (3);
+
+ if (printf ("\n") < 0)
+ exit (3);
+
+ /* TRANSLATORS: When available, an actual copyright character
+ (circle-c) should be used in preference to "(C)". */
+ if (printf (_("\
+Copyright (C) %s Free Software Foundation, Inc.\n"), "2014") < 0)
+ exit (3);
+ if (fputs (_("\
+License GPLv3+: GNU GPL version 3 or later\n\
+.\n\
+This is free software: you are free to change and redistribute it.\n\
+There is NO WARRANTY, to the extent permitted by law.\n"), stdout) < 0)
+ exit (3);
+ /* TRANSLATORS: When available, please use the proper diacritics for
+ names such as this one. See en_US.po for reference. */
+ if (fputs (_("\nOriginally written by Hrvoje Niksic .\n"),
+ stdout) < 0)
+ exit (3);
+ if (fputs (_("Please send bug reports and questions to .\n"),
+ stdout) < 0)
+ exit (3);
+
exit (0);
}
-
+
+static char *program_name; /* Needed by lib/error.c. */
+char *program_argstring; /* Needed by wget_warc.c. */
+
int
-main (int argc, char *const *argv)
+main (int argc, char **argv)
{
char **url, **t;
int i, ret, longindex;
- int nurl, status;
+ int nurl;
bool append_to_log = false;
+ total_downloaded_bytes = 0;
+
+ program_name = argv[0];
+
+ struct ptimer *timer = ptimer_new ();
+ double start_time = ptimer_measure (timer);
+
i18n_initialize ();
/* Construct the name of the executable, without the directory part. */
+#ifdef __VMS
+ /* On VMS, lose the "dev:[dir]" prefix and the ".EXE;nnn" suffix. */
+ exec_name = vms_basename (argv[0]);
+#else /* def __VMS */
exec_name = strrchr (argv[0], PATH_SEPARATOR);
if (!exec_name)
exec_name = argv[0];
else
++exec_name;
+#endif /* def __VMS [else] */
#ifdef WINDOWS
/* Drop extension (typically .EXE) from executable filename. */
- windows_main (&argc, (char **) argv, (char **) &exec_name);
+ windows_main ((char **) &exec_name);
#endif
- /* Set option defaults; read the system wgetrc and ~/.wgetrc. */
- initialize ();
+ /* Construct the arguments string. */
+ int argstring_length = 1;
+ for (i = 1; i < argc; i++)
+ argstring_length += strlen (argv[i]) + 2 + 1;
+ char *p = program_argstring = malloc (argstring_length * sizeof (char));
+ if (p == NULL)
+ {
+ fprintf (stderr, _("Memory allocation problem\n"));
+ exit (2);
+ }
+ for (i = 1; i < argc; i++)
+ {
+ *p++ = '"';
+ int arglen = strlen (argv[i]);
+ memcpy (p, argv[i], arglen);
+ p += arglen;
+ *p++ = '"';
+ *p++ = ' ';
+ }
+ *p = '\0';
+
+ /* Load the hard-coded defaults. */
+ defaults ();
init_switches ();
+
+ /* This separate getopt_long is needed to find the user config file
+ option ("--config") and parse it before the other user options. */
+ longindex = -1;
+ int retconf;
+ bool use_userconfig = false;
+ bool noconfig = false;
+
+ while ((retconf = getopt_long (argc, argv,
+ short_options, long_options, &longindex)) != -1)
+ {
+ int confval;
+ struct cmdline_option *config_opt;
+
+ /* There is no short option for "--config". */
+ if (longindex >= 0)
+ {
+ confval = long_options[longindex].val;
+ config_opt = &option_data[confval & ~BOOLEAN_NEG_MARKER];
+ if (strcmp (config_opt->long_name, "no-config") == 0)
+ {
+ noconfig = true;
+ break;
+ }
+ else if (strcmp (config_opt->long_name, "config") == 0)
+ {
+ bool userrc_ret = true;
+ userrc_ret &= run_wgetrc (optarg);
+ use_userconfig = true;
+ if (userrc_ret)
+ break;
+ else
+ {
+ fprintf (stderr, _("Exiting due to error in %s\n"), optarg);
+ exit (2);
+ }
+ }
+ }
+ }
+
+ /* If the user did not specify a config, read the system wgetrc and ~/.wgetrc. */
+ if (noconfig == false && use_userconfig == false)
+ initialize ();
+
+ opterr = 0;
+ optind = 0;
+
longindex = -1;
while ((ret = getopt_long (argc, argv,
- short_options, long_options, &longindex)) != -1)
+ short_options, long_options, &longindex)) != -1)
{
int val;
- struct cmdline_option *opt;
+ struct cmdline_option *cmdopt;
/* If LONGINDEX is unchanged, it means RET is referring a short
- option. */
+ option. */
if (longindex == -1)
- {
- if (ret == '?')
- {
- print_usage ();
- printf ("\n");
- printf (_("Try `%s --help' for more options.\n"), exec_name);
- exit (2);
- }
- /* Find the short option character in the mapping. */
- longindex = optmap[ret - 32];
- }
+ {
+ if (ret == '?')
+ {
+ print_usage (1);
+ fprintf (stderr, "\n");
+ fprintf (stderr, _("Try `%s --help' for more options.\n"),
+ exec_name);
+ exit (2);
+ }
+ /* Find the short option character in the mapping. */
+ longindex = optmap[ret - 32];
+ }
val = long_options[longindex].val;
/* Use the retrieved value to locate the option in the
- option_data array, and to see if we're dealing with the
- negated "--no-FOO" variant of the boolean option "--foo". */
- opt = &option_data[val & ~BOOLEAN_NEG_MARKER];
- switch (opt->type)
- {
- case OPT_VALUE:
- setoptval (opt->data, optarg, opt->long_name);
- break;
- case OPT_BOOLEAN:
- if (optarg)
- /* The user has specified a value -- use it. */
- setoptval (opt->data, optarg, opt->long_name);
- else
- {
- /* NEG is true for `--no-FOO' style boolean options. */
- bool neg = !!(val & BOOLEAN_NEG_MARKER);
- setoptval (opt->data, neg ? "0" : "1", opt->long_name);
- }
- break;
- case OPT_FUNCALL:
- {
- void (*func) (void) = (void (*) (void)) opt->data;
- func ();
- }
- break;
- case OPT__APPEND_OUTPUT:
- setoptval ("logfile", optarg, opt->long_name);
- append_to_log = true;
- break;
- case OPT__EXECUTE:
- run_command (optarg);
- break;
- case OPT__NO:
- {
- /* We support real --no-FOO flags now, but keep these
- short options for convenience and backward
- compatibility. */
- char *p;
- for (p = optarg; *p; p++)
- switch (*p)
- {
- case 'v':
- setoptval ("verbose", "0", opt->long_name);
- break;
- case 'H':
- setoptval ("addhostdir", "0", opt->long_name);
- break;
- case 'd':
- setoptval ("dirstruct", "0", opt->long_name);
- break;
- case 'c':
- setoptval ("noclobber", "1", opt->long_name);
- break;
- case 'p':
- setoptval ("noparent", "1", opt->long_name);
- break;
- default:
- printf (_("%s: illegal option -- `-n%c'\n"), exec_name, *p);
- print_usage ();
- printf ("\n");
- printf (_("Try `%s --help' for more options.\n"), exec_name);
- exit (1);
- }
- break;
- }
- case OPT__PARENT:
- case OPT__CLOBBER:
- {
- /* The wgetrc commands are named noparent and noclobber,
- so we must revert the meaning of the cmdline options
- before passing the value to setoptval. */
- bool flag = true;
- if (optarg)
- flag = (*optarg == '1' || TOLOWER (*optarg) == 'y'
- || (TOLOWER (optarg[0]) == 'o'
- && TOLOWER (optarg[1]) == 'n'));
- setoptval (opt->type == OPT__PARENT ? "noparent" : "noclobber",
- flag ? "0" : "1", opt->long_name);
- break;
- }
- case OPT__DONT_REMOVE_LISTING:
- setoptval ("removelisting", "0", opt->long_name);
- break;
- }
+ option_data array, and to see if we're dealing with the
+ negated "--no-FOO" variant of the boolean option "--foo". */
+ cmdopt = &option_data[val & ~BOOLEAN_NEG_MARKER];
+ switch (cmdopt->type)
+ {
+ case OPT_VALUE:
+ setoptval (cmdopt->data, optarg, cmdopt->long_name);
+ break;
+ case OPT_BOOLEAN:
+ if (optarg)
+ /* The user has specified a value -- use it. */
+ setoptval (cmdopt->data, optarg, cmdopt->long_name);
+ else
+ {
+ /* NEG is true for `--no-FOO' style boolean options. */
+ bool neg = !!(val & BOOLEAN_NEG_MARKER);
+ setoptval (cmdopt->data, neg ? "0" : "1", cmdopt->long_name);
+ }
+ break;
+ case OPT_FUNCALL:
+ {
+ void (*func) (void) = (void (*) (void)) cmdopt->data;
+ func ();
+ }
+ break;
+ case OPT__APPEND_OUTPUT:
+ setoptval ("logfile", optarg, cmdopt->long_name);
+ append_to_log = true;
+ break;
+ case OPT__EXECUTE:
+ run_command (optarg);
+ break;
+ case OPT__NO:
+ {
+ /* We support real --no-FOO flags now, but keep these
+ short options for convenience and backward
+ compatibility. */
+ for (p = optarg; p && *p; p++)
+ switch (*p)
+ {
+ case 'v':
+ setoptval ("verbose", "0", cmdopt->long_name);
+ break;
+ case 'H':
+ setoptval ("addhostdir", "0", cmdopt->long_name);
+ break;
+ case 'd':
+ setoptval ("dirstruct", "0", cmdopt->long_name);
+ break;
+ case 'c':
+ setoptval ("noclobber", "1", cmdopt->long_name);
+ break;
+ case 'p':
+ setoptval ("noparent", "1", cmdopt->long_name);
+ break;
+ default:
+ fprintf (stderr, _("%s: illegal option -- `-n%c'\n"),
+ exec_name, *p);
+ print_usage (1);
+ fprintf (stderr, "\n");
+ fprintf (stderr, _("Try `%s --help' for more options.\n"),
+ exec_name);
+ exit (1);
+ }
+ break;
+ }
+ case OPT__PARENT:
+ case OPT__CLOBBER:
+ {
+ /* The wgetrc commands are named noparent and noclobber,
+ so we must revert the meaning of the cmdline options
+ before passing the value to setoptval. */
+ bool flag = true;
+ if (optarg)
+ flag = (*optarg == '1' || c_tolower (*optarg) == 'y'
+ || (c_tolower (optarg[0]) == 'o'
+ && c_tolower (optarg[1]) == 'n'));
+ setoptval (cmdopt->type == OPT__PARENT ? "noparent" : "noclobber",
+ flag ? "0" : "1", cmdopt->long_name);
+ break;
+ }
+ case OPT__DONT_REMOVE_LISTING:
+ setoptval ("removelisting", "0", cmdopt->long_name);
+ break;
+ }
longindex = -1;
}
+ nurl = argc - optind;
+
+ /* If we do not have Debug support compiled in AND Wget is invoked with the
+ * --debug switch, instead of failing, we silently turn it into a no-op. For
+ * this no-op, we explicitly set opt.debug to false and hence none of the
+ * Debug output messages will be printed.
+ */
+#ifndef ENABLE_DEBUG
+ if (opt.debug)
+ {
+ fprintf (stderr, _("Debugging support not compiled in. "
+ "Ignoring --debug flag.\n"));
+ opt.debug = false;
+ }
+#endif
+
/* All user options have now been processed, so it's now safe to do
interoption dependency checks. */
+ if (opt.noclobber && opt.convert_links)
+ {
+ fprintf (stderr,
+ _("Both --no-clobber and --convert-links were specified,"
+ " only --convert-links will be used.\n"));
+ opt.noclobber = false;
+ }
+
if (opt.reclevel == 0)
- opt.reclevel = INFINITE_RECURSION; /* see recur.h for commentary on this */
+ opt.reclevel = INFINITE_RECURSION; /* see recur.h for commentary */
+
+ if (opt.spider || opt.delete_after)
+ opt.no_dirstruct = true;
if (opt.page_requisites && !opt.recursive)
{
/* Don't set opt.recursive here because it would confuse the FTP
- code. Instead, call retrieve_tree below when either
- page_requisites or recursive is requested. */
+ code. Instead, call retrieve_tree below when either
+ page_requisites or recursive is requested. */
opt.reclevel = 0;
if (!opt.no_dirstruct)
- opt.dirstruct = 1; /* normally handled by cmd_spec_recursive() */
+ opt.dirstruct = 1; /* normally handled by cmd_spec_recursive() */
}
if (opt.verbose == -1)
opt.verbose = !opt.quiet;
+ if (opt.verbose == 1)
+ opt.show_progress = true;
+
/* Sanity checks. */
if (opt.verbose && opt.quiet)
{
- printf (_("Can't be verbose and quiet at the same time.\n"));
- print_usage ();
+ fprintf (stderr, _("Can't be verbose and quiet at the same time.\n"));
+ print_usage (1);
exit (1);
}
if (opt.timestamping && opt.noclobber)
{
- printf (_("\
+ fprintf (stderr, _("\
Can't timestamp and not clobber old files at the same time.\n"));
- print_usage ();
+ print_usage (1);
exit (1);
}
#ifdef ENABLE_IPV6
if (opt.ipv4_only && opt.ipv6_only)
{
- printf (_("Cannot specify both --inet4-only and --inet6-only.\n"));
- print_usage ();
+ fprintf (stderr,
+ _("Cannot specify both --inet4-only and --inet6-only.\n"));
+ print_usage (1);
exit (1);
}
#endif
+ if (opt.output_document)
+ {
+ if (opt.convert_links
+ && (nurl > 1 || opt.page_requisites || opt.recursive))
+ {
+ fputs (_("\
+Cannot specify both -k and -O if multiple URLs are given, or in combination\n\
+with -p or -r. See the manual for details.\n\n"), stderr);
+ print_usage (1);
+ exit (1);
+ }
+ if (opt.page_requisites
+ || opt.recursive)
+ {
+ logprintf (LOG_NOTQUIET, "%s", _("\
+WARNING: combining -O with -r or -p will mean that all downloaded content\n\
+will be placed in the single file you specified.\n\n"));
+ }
+ if (opt.timestamping)
+ {
+ logprintf (LOG_NOTQUIET, "%s", _("\
+WARNING: timestamping does nothing in combination with -O. See the manual\n\
+for details.\n\n"));
+ opt.timestamping = false;
+ }
+ if (opt.noclobber && file_exists_p(opt.output_document))
+ {
+ /* Check if output file exists; if it does, exit. */
+ logprintf (LOG_VERBOSE,
+ _("File `%s' already there; not retrieving.\n"),
+ opt.output_document);
+ exit(1);
+ }
+ }
+
+ if (opt.warc_filename != 0)
+ {
+ if (opt.noclobber)
+ {
+ fprintf (stderr,
+ _("WARC output does not work with --no-clobber, "
+ "--no-clobber will be disabled.\n"));
+ opt.noclobber = false;
+ }
+ if (opt.timestamping)
+ {
+ fprintf (stderr,
+ _("WARC output does not work with timestamping, "
+ "timestamping will be disabled.\n"));
+ opt.timestamping = false;
+ }
+ if (opt.spider)
+ {
+ fprintf (stderr,
+ _("WARC output does not work with --spider.\n"));
+ exit (1);
+ }
+ if (opt.always_rest || opt.start_pos >= 0)
+ {
+ fprintf (stderr,
+ _("WARC output does not work with --continue or"
+ " --start-pos, they will be disabled.\n"));
+ opt.always_rest = false;
+ opt.start_pos = -1;
+ }
+ if (opt.warc_cdx_dedup_filename != 0 && !opt.warc_digests_enabled)
+ {
+ fprintf (stderr,
+ _("Digests are disabled; WARC deduplication will "
+ "not find duplicate records.\n"));
+ }
+ if (opt.warc_keep_log)
+ {
+ opt.progress_type = xstrdup ("dot");
+ }
+ }
+
+ if (opt.ask_passwd && opt.passwd)
+ {
+ fprintf (stderr,
+ _("Cannot specify both --ask-password and --password.\n"));
+ print_usage (1);
+ exit (1);
+ }
+
+ if (opt.start_pos >= 0 && opt.always_rest)
+ {
+ fprintf (stderr,
+ _("Specifying both --start-pos and --continue is not "
+ "recommended; --continue will be disabled.\n"));
+ opt.always_rest = false;
+ }
- nurl = argc - optind;
if (!nurl && !opt.input_filename)
{
/* No URL specified. */
- printf (_("%s: missing URL\n"), exec_name);
- print_usage ();
- printf ("\n");
+ fprintf (stderr, _("%s: missing URL\n"), exec_name);
+ print_usage (1);
+ fprintf (stderr, "\n");
/* #### Something nicer should be printed here -- similar to the
- pre-1.5 `--help' page. */
- printf (_("Try `%s --help' for more options.\n"), exec_name);
+ pre-1.5 `--help' page. */
+ fprintf (stderr, _("Try `%s --help' for more options.\n"), exec_name);
exit (1);
}
+ /* Compile the regular expressions. */
+ switch (opt.regex_type)
+ {
+#ifdef HAVE_LIBPCRE
+ case regex_type_pcre:
+ opt.regex_compile_fun = compile_pcre_regex;
+ opt.regex_match_fun = match_pcre_regex;
+ break;
+#endif
+
+ case regex_type_posix:
+ default:
+ opt.regex_compile_fun = compile_posix_regex;
+ opt.regex_match_fun = match_posix_regex;
+ break;
+ }
+ if (opt.acceptregex_s)
+ {
+ opt.acceptregex = opt.regex_compile_fun (opt.acceptregex_s);
+ if (!opt.acceptregex)
+ exit (1);
+ }
+ if (opt.rejectregex_s)
+ {
+ opt.rejectregex = opt.regex_compile_fun (opt.rejectregex_s);
+ if (!opt.rejectregex)
+ exit (1);
+ }
+ if (opt.post_data || opt.post_file_name)
+ {
+ if (opt.post_data && opt.post_file_name)
+ {
+ fprintf (stderr, _("You cannot specify both --post-data and --post-file.\n"));
+ exit (1);
+ }
+ else if (opt.method)
+ {
+ fprintf (stderr, _("You cannot use --post-data or --post-file along with --method. "
+ "--method expects data through --body-data and --body-file options"));
+ exit (1);
+ }
+ }
+ if (opt.body_data || opt.body_file)
+ {
+ if (!opt.method)
+ {
+ fprintf (stderr, _("You must specify a method through --method=HTTPMethod "
+ "to use with --body-data or --body-file.\n"));
+ exit (1);
+ }
+ else if (opt.body_data && opt.body_file)
+ {
+ fprintf (stderr, _("You cannot specify both --body-data and --body-file.\n"));
+ exit (1);
+ }
+ }
+
+ /* Set various options as required for opt.method. */
+
+ /* When user specifies HEAD as the method, we do not wish to download any
+ files. Hence, set wget to run in spider mode. */
+ if (opt.method && strcasecmp (opt.method, "HEAD") == 0)
+ setoptval ("spider", "1", "spider");
+
+ /* Convert post_data to body-data and post_file_name to body-file options.
+ This is required so as to remove redundant code later on in gethttp().
+ The --post-data and --post-file options may also be removed in
+ the future hence it makes sense to convert them to aliases for
+ the more generic --method options.
+ This MUST occur only after the sanity checks so as to prevent the
+ user from setting both post and body options simultaneously.
+ */
+ if (opt.post_data || opt.post_file_name)
+ {
+ setoptval ("method", "POST", "method");
+ if (opt.post_data)
+ {
+ setoptval ("bodydata", opt.post_data, "body-data");
+ opt.post_data = NULL;
+ }
+ else
+ {
+ setoptval ("bodyfile", opt.post_file_name, "body-file");
+ opt.post_file_name = NULL;
+ }
+ }
+
+#ifdef ENABLE_IRI
+ if (opt.enable_iri)
+ {
+ if (opt.locale && !check_encoding_name (opt.locale))
+ opt.locale = NULL;
+
+ if (!opt.locale)
+ opt.locale = find_locale ();
+
+ if (opt.encoding_remote && !check_encoding_name (opt.encoding_remote))
+ opt.encoding_remote = NULL;
+ }
+#else
+ memset (&dummy_iri, 0, sizeof (dummy_iri));
+ if (opt.enable_iri || opt.locale || opt.encoding_remote)
+ {
+ /* sXXXav : be more specific... */
+ fprintf (stderr, _("This version does not have support for IRIs\n"));
+ exit(1);
+ }
+#endif
+
+ if (opt.ask_passwd)
+ {
+ opt.passwd = prompt_for_password ();
+
+ if (opt.passwd == NULL || opt.passwd[0] == '\0')
+ exit (1);
+ }
+
+#ifdef USE_WATT32
+ if (opt.wdebug)
+ dbug_init();
+ sock_init();
+#else
if (opt.background)
fork_to_background ();
+#endif
/* Initialize progress. Have to do this after the options are
processed so we know where the log file is. */
- if (opt.verbose)
+ if (opt.show_progress)
set_progress_implementation (opt.progress_type);
/* Fill in the arguments. */
url = alloca_array (char *, nurl + 1);
+ if (url == NULL)
+ {
+ fprintf (stderr, _("Memory allocation problem\n"));
+ exit (2);
+ }
for (i = 0; i < nurl; i++, optind++)
{
char *rewritten = rewrite_shorthand_url (argv[optind]);
if (rewritten)
- url[i] = rewritten;
+ url[i] = rewritten;
else
- url[i] = xstrdup (argv[optind]);
+ url[i] = xstrdup (argv[optind]);
}
url[i] = NULL;
/* Initialize logging. */
log_init (opt.lfilename, append_to_log);
- DEBUGP (("DEBUG output created by Wget %s on %s.\n\n", version_string,
- OS_TYPE));
+ /* Open WARC file. */
+ if (opt.warc_filename != 0)
+ warc_init ();
+
+ DEBUGP (("DEBUG output created by Wget %s on %s.\n\n",
+ version_string, OS_TYPE));
/* Open the output filename if necessary. */
+
+/* 2005-04-17 SMS.
+ Note that having the output_stream ("-O") file opened here for an FTP
+ URL rather than in getftp() (ftp.c) (and the http equivalent) rather
+ limits the ability in VMS to open the file differently for ASCII
+ versus binary FTP there. (Of course, doing it here allows a open
+ failure to be detected immediately, without first connecting to the
+ server.)
+*/
if (opt.output_document)
{
if (HYPHENP (opt.output_document))
- output_stream = stdout;
+ {
+#ifdef WINDOWS
+ _setmode (_fileno (stdout), _O_BINARY);
+#endif
+ output_stream = stdout;
+ }
else
- {
- struct_fstat st;
- output_stream = fopen (opt.output_document,
- opt.always_rest ? "ab" : "wb");
- if (output_stream == NULL)
- {
- perror (opt.output_document);
- exit (1);
- }
- if (fstat (fileno (output_stream), &st) == 0 && S_ISREG (st.st_mode))
- output_stream_regular = true;
- }
+ {
+ struct_fstat st;
+
+#ifdef __VMS
+/* Common fopen() optional arguments:
+ sequential access only, access callback function.
+*/
+# define FOPEN_OPT_ARGS , "fop=sqo", "acc", acc_cb, &open_id
+ int open_id = 7;
+#else /* def __VMS */
+# define FOPEN_OPT_ARGS
+#endif /* def __VMS [else] */
+
+ output_stream = fopen (opt.output_document,
+ opt.always_rest ? "ab" : "wb"
+ FOPEN_OPT_ARGS);
+ if (output_stream == NULL)
+ {
+ perror (opt.output_document);
+ exit (1);
+ }
+ if (fstat (fileno (output_stream), &st) == 0 && S_ISREG (st.st_mode))
+ output_stream_regular = true;
+ }
+ if (!output_stream_regular && opt.convert_links)
+ {
+ fprintf (stderr, _("-k can be used together with -O only if \
+outputting to a regular file.\n"));
+ print_usage (1);
+ exit(1);
+ }
}
+#ifdef __VMS
+ /* Set global ODS5 flag according to the specified destination (if
+ any), otherwise according to the current default device.
+ */
+ if (output_stream == NULL)
+ set_ods5_dest( "SYS$DISK");
+ else if (output_stream != stdout)
+ set_ods5_dest( opt.output_document);
+#endif /* def __VMS */
+
#ifdef WINDOWS
ws_startup ();
#endif
@@ -930,57 +1624,107 @@ Can't timestamp and not clobber old files at the same time.\n"));
signal (SIGWINCH, progress_handle_sigwinch);
#endif
- status = RETROK; /* initialize it, just-in-case */
/* Retrieve the URLs from argument list. */
for (t = url; *t; t++)
{
char *filename = NULL, *redirected_URL = NULL;
- int dt;
-
- if ((opt.recursive || opt.page_requisites)
- && url_scheme (*t) != SCHEME_FTP)
- status = retrieve_tree (*t);
+ int dt, url_err;
+ /* Need to do a new struct iri every time, because
+ * retrieve_url may modify it in some circumstances,
+ * currently. */
+ struct iri *iri = iri_new ();
+ struct url *url_parsed;
+
+ set_uri_encoding (iri, opt.locale, true);
+ url_parsed = url_parse (*t, &url_err, iri, true);
+
+ if (!url_parsed)
+ {
+ char *error = url_error (*t, url_err);
+ logprintf (LOG_NOTQUIET, "%s: %s.\n",*t, error);
+ xfree (error);
+ inform_exit_status (URLERROR);
+ }
else
- status = retrieve_url (*t, &filename, &redirected_URL, NULL, &dt);
-
- if (opt.delete_after && file_exists_p(filename))
- {
- DEBUGP (("Removing file due to --delete-after in main():\n"));
- logprintf (LOG_VERBOSE, _("Removing %s.\n"), filename);
- if (unlink (filename))
- logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
- }
-
- xfree_null (redirected_URL);
- xfree_null (filename);
+ {
+ if ((opt.recursive || opt.page_requisites)
+ && (url_scheme (*t) != SCHEME_FTP || url_uses_proxy (url_parsed)))
+ {
+ int old_follow_ftp = opt.follow_ftp;
+
+ /* Turn opt.follow_ftp on in case of recursive FTP retrieval */
+ if (url_scheme (*t) == SCHEME_FTP)
+ opt.follow_ftp = 1;
+
+ retrieve_tree (url_parsed, NULL);
+
+ opt.follow_ftp = old_follow_ftp;
+ }
+ else
+ {
+ retrieve_url (url_parsed, *t, &filename, &redirected_URL, NULL,
+ &dt, opt.recursive, iri, true);
+ }
+
+ if (opt.delete_after && filename != NULL && file_exists_p (filename))
+ {
+ DEBUGP (("Removing file due to --delete-after in main():\n"));
+ logprintf (LOG_VERBOSE, _("Removing %s.\n"), filename);
+ if (unlink (filename))
+ logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
+ }
+ xfree_null (redirected_URL);
+ xfree_null (filename);
+ url_free (url_parsed);
+ }
+ iri_free (iri);
}
/* And then from the input file, if any. */
if (opt.input_filename)
{
int count;
+ int status;
status = retrieve_from_file (opt.input_filename, opt.force_html, &count);
+ inform_exit_status (status);
if (!count)
- logprintf (LOG_NOTQUIET, _("No URLs found in %s.\n"),
- opt.input_filename);
+ logprintf (LOG_NOTQUIET, _("No URLs found in %s.\n"),
+ opt.input_filename);
}
+
+ /* Print broken links. */
+ if (opt.recursive && opt.spider)
+ print_broken_links ();
+
/* Print the downloaded sum. */
- if (opt.recursive || opt.page_requisites
- || nurl > 1
- || (opt.input_filename && total_downloaded_bytes != 0))
+ if ((opt.recursive || opt.page_requisites
+ || nurl > 1
+ || (opt.input_filename && total_downloaded_bytes != 0))
+ &&
+ total_downloaded_bytes != 0)
{
+ double end_time = ptimer_measure (timer);
+ ptimer_destroy (timer);
+
+ char *wall_time = xstrdup (secs_to_human_time (end_time - start_time));
+ char *download_time = xstrdup (secs_to_human_time (total_download_time));
logprintf (LOG_NOTQUIET,
- _("FINISHED --%s--\nDownloaded: %d files, %s in %s (%s)\n"),
- time_str (NULL),
- opt.numurls,
+ _("FINISHED --%s--\nTotal wall clock time: %s\n"
+ "Downloaded: %d files, %s in %s (%s)\n"),
+ datetime_str (time (NULL)),
+ wall_time,
+ numurls,
human_readable (total_downloaded_bytes),
- secs_to_human_time (total_download_time),
+ download_time,
retr_rate (total_downloaded_bytes, total_download_time));
+ xfree (wall_time);
+ xfree (download_time);
+
/* Print quota warning, if exceeded. */
if (opt.quota && total_downloaded_bytes > opt.quota)
- logprintf (LOG_NOTQUIET,
- _("Download quota of %s EXCEEDED!\n"),
- human_readable (opt.quota));
+ logprintf (LOG_NOTQUIET,
+ _("Download quota of %s EXCEEDED!\n"),
+ human_readable (opt.quota));
}
if (opt.cookies_output)
@@ -989,19 +1733,11 @@ Can't timestamp and not clobber old files at the same time.\n"));
if (opt.convert_links && !opt.delete_after)
convert_all_links ();
- log_close ();
- for (i = 0; i < nurl; i++)
- xfree (url[i]);
cleanup ();
-#ifdef DEBUG_MALLOC
- print_malloc_debug_stats ();
-#endif
- if (status == RETROK)
- return 0;
- else
- return 1;
+ exit (get_exit_status ());
}
+#endif /* TESTING */
#if defined(SIGHUP) || defined(SIGUSR1)
@@ -1021,10 +1757,14 @@ static void
redirect_output_signal (int sig)
{
const char *signal_name = (sig == SIGHUP ? "SIGHUP" :
- (sig == SIGUSR1 ? "SIGUSR1" :
- "WTF?!"));
+ (sig == SIGUSR1 ? "SIGUSR1" :
+ "WTF?!"));
log_request_redirect_output (signal_name);
progress_schedule_redirect ();
signal (sig, redirect_output_signal);
}
#endif
+
+/*
+ * vim: et ts=2 sw=2
+ */