X-Git-Url: http://sjero.net/git/?p=wget;a=blobdiff_plain;f=src%2Fmain.c;h=8414f5e54d0e85caaa203405b51fa48b9be0cb37;hp=a12aaf1b43e2ebde3ba36427b521acb26f6f9c7f;hb=42c78fdd71c311cf96210b709ec0a18ef45ef87f;hpb=b9b510ca5f9c13f8d3f129faae61f324f5c816d5 diff --git a/src/main.c b/src/main.c index a12aaf1b..8414f5e5 100644 --- a/src/main.c +++ b/src/main.c @@ -56,7 +56,6 @@ as that of the covered work. */ #include "http.h" /* for save_cookies */ #include "ptimer.h" #include "warc.h" -#include "closeout.h" #include #include #include @@ -158,6 +157,7 @@ struct cmdline_option { static struct cmdline_option option_data[] = { { "accept", 'A', OPT_VALUE, "accept", -1 }, + { "accept-regex", 0, OPT_VALUE, "acceptregex", -1 }, { "adjust-extension", 'E', OPT_BOOLEAN, "adjustextension", -1 }, { "append-output", 'a', OPT__APPEND_OUTPUT, NULL, required_argument }, { "ask-password", 0, OPT_BOOLEAN, "askpassword", -1 }, @@ -167,7 +167,8 @@ static struct cmdline_option option_data[] = { "backups", 0, OPT_BOOLEAN, "backups", -1 }, { "base", 'B', OPT_VALUE, "base", -1 }, { "bind-address", 0, OPT_VALUE, "bindaddress", -1 }, - { "bits", 0, OPT_BOOLEAN, "bits", -1 }, + { "body-data", 0, OPT_VALUE, "bodydata", -1 }, + { "body-file", 0, OPT_VALUE, "bodyfile", -1 }, { IF_SSL ("ca-certificate"), 0, OPT_VALUE, "cacertificate", -1 }, { IF_SSL ("ca-directory"), 0, OPT_VALUE, "cadirectory", -1 }, { "cache", 0, OPT_BOOLEAN, "cache", -1 }, @@ -216,6 +217,7 @@ static struct cmdline_option option_data[] = { "http-passwd", 0, OPT_VALUE, "httppassword", -1 }, /* deprecated */ { "http-password", 0, OPT_VALUE, "httppassword", -1 }, { "http-user", 0, OPT_VALUE, "httpuser", -1 }, + { IF_SSL ("https-only"), 0, OPT_BOOLEAN, "httpsonly", -1 }, { "ignore-case", 0, OPT_BOOLEAN, "ignorecase", -1 }, { "ignore-length", 0, OPT_BOOLEAN, "ignorelength", -1 }, { "ignore-tags", 0, OPT_VALUE, "ignoretags", -1 }, @@ -232,6 +234,7 @@ static struct cmdline_option option_data[] = { "load-cookies", 0, OPT_VALUE, "loadcookies", -1 }, { "local-encoding", 0, OPT_VALUE, "localencoding", -1 }, { "max-redirect", 0, OPT_VALUE, "maxredirect", -1 }, + { "method", 0, OPT_VALUE, "method", -1 }, { "mirror", 'm', OPT_BOOLEAN, "mirror", -1 }, { "no", 'n', OPT__NO, NULL, required_argument }, { "no-clobber", 0, OPT_BOOLEAN, "noclobber", -1 }, @@ -262,10 +265,13 @@ static struct cmdline_option option_data[] = { "read-timeout", 0, OPT_VALUE, "readtimeout", -1 }, { "recursive", 'r', OPT_BOOLEAN, "recursive", -1 }, { "referer", 0, OPT_VALUE, "referer", -1 }, + { "regex-type", 0, OPT_VALUE, "regextype", -1 }, { "reject", 'R', OPT_VALUE, "reject", -1 }, + { "reject-regex", 0, OPT_VALUE, "rejectregex", -1 }, { "relative", 'L', OPT_BOOLEAN, "relativeonly", -1 }, { "remote-encoding", 0, OPT_VALUE, "remoteencoding", -1 }, { "remove-listing", 0, OPT_BOOLEAN, "removelisting", -1 }, + { "report-speed", 0, OPT_BOOLEAN, "reportspeed", -1 }, { "restrict-file-names", 0, OPT_BOOLEAN, "restrictfilenames", -1 }, { "retr-symlinks", 0, OPT_BOOLEAN, "retrsymlinks", -1 }, { "retry-connrefused", 0, OPT_BOOLEAN, "retryconnrefused", -1 }, @@ -457,6 +463,8 @@ Logging and input file:\n"), -v, --verbose be verbose (this is the default).\n"), N_("\ -nv, --no-verbose turn off verboseness, without being quiet.\n"), + N_("\ + --report-speed=TYPE Output bandwidth as TYPE. TYPE can be bits.\n"), N_("\ -i, --input-file=FILE download URLs found in local or external FILE.\n"), N_("\ @@ -605,6 +613,12 @@ HTTP options:\n"), --post-data=STRING use the POST method; send STRING as the data.\n"), N_("\ --post-file=FILE use the POST method; send contents of FILE.\n"), + N_("\ + --method=HTTPMethod use method \"HTTPMethod\" in the header.\n"), + N_("\ + --body-data=STRING Send STRING as data. --method MUST be set.\n"), + N_("\ + --body-file=FILE Send contents of FILE. --method MUST be set.\n"), N_("\ --content-disposition honor the Content-Disposition header when\n\ choosing local file names (EXPERIMENTAL).\n"), @@ -622,6 +636,8 @@ HTTPS (SSL/TLS) options:\n"), N_("\ --secure-protocol=PR choose secure protocol, one of auto, SSLv2,\n\ SSLv3, and TLSv1.\n"), + N_("\ + --https-only only follow secure HTTPS links\n"), N_("\ --no-check-certificate don't validate the server's certificate.\n"), N_("\ @@ -701,6 +717,9 @@ Recursive download:\n"), N_("\ -k, --convert-links make links in downloaded HTML or CSS point to\n\ local files.\n"), + N_("\ + --backups=N before writing file X, rotate up to N backup files.\n"), + #ifdef __VMS N_("\ -K, --backup-converted before converting file X, back up as X_orig.\n"), @@ -722,6 +741,17 @@ Recursive accept/reject:\n"), -A, --accept=LIST comma-separated list of accepted extensions.\n"), N_("\ -R, --reject=LIST comma-separated list of rejected extensions.\n"), + N_("\ + --accept-regex=REGEX regex matching accepted URLs.\n"), + N_("\ + --reject-regex=REGEX regex matching rejected URLs.\n"), +#ifdef HAVE_LIBPCRE + N_("\ + --regex-type=TYPE regex type (posix|pcre).\n"), +#else + N_("\ + --regex-type=TYPE regex type (posix).\n"), +#endif N_("\ -D, --domains=LIST comma-separated list of accepted domains.\n"), N_("\ @@ -746,12 +776,6 @@ Recursive accept/reject:\n"), N_("\ -np, --no-parent don't ascend to the parent directory.\n"), "\n", - - N_("\ -Output format:\n"), - N_("\ - --bits Output bandwidth in bits.\n"), - "\n", N_("Mail bug reports and suggestions to .\n") }; @@ -819,15 +843,16 @@ format_and_print_line (const char *prefix, const char *line, assert (prefix != NULL); assert (line != NULL); + assert (line_length > TABULATION); line_dup = xstrdup (line); - if (line_length <= 0) - line_length = MAX_CHARS_PER_LINE - TABULATION; - if (printf ("%s", prefix) < 0) return -1; - remaining_chars = line_length; + + /* Wrap to new line after prefix. */ + remaining_chars = 0; + /* We break on spaces. */ token = strtok (line_dup, " "); while (token != NULL) @@ -835,7 +860,7 @@ format_and_print_line (const char *prefix, const char *line, /* If however a token is much larger than the maximum line length, all bets are off and we simply print the token on the next line. */ - if (remaining_chars <= strlen (token)) + if (remaining_chars <= (int) strlen (token)) { if (printf ("\n%*c", TABULATION, ' ') < 0) return -1; @@ -909,7 +934,7 @@ print_version (void) #ifdef ENABLE_NLS if (format_and_print_line (locale_title, - LOCALEDIR, + LOCALEDIR, MAX_CHARS_PER_LINE) < 0) exit (3); #endif /* def ENABLE_NLS */ @@ -972,8 +997,6 @@ main (int argc, char **argv) i18n_initialize (); - atexit (close_stdout); - /* Construct the name of the executable, without the directory part. */ #ifdef __VMS /* On VMS, lose the "dev:[dir]" prefix and the ".EXE;nnn" suffix. */ @@ -1027,7 +1050,6 @@ main (int argc, char **argv) short_options, long_options, &longindex)) != -1) { int confval; - bool userrc_ret = true; struct cmdline_option *config_opt; /* There is no short option for "--config". */ @@ -1037,16 +1059,17 @@ main (int argc, char **argv) config_opt = &option_data[confval & ~BOOLEAN_NEG_MARKER]; if (strcmp (config_opt->long_name, "config") == 0) { + bool userrc_ret = true; userrc_ret &= run_wgetrc (optarg); use_userconfig = true; + if (userrc_ret) + break; + else + { + fprintf (stderr, _("Exiting due to error in %s\n"), optarg); + exit (2); + } } - if (!userrc_ret) - { - fprintf (stderr, "Exiting due to error in %s\n", optarg); - exit (2); - } - else - break; } } @@ -1181,7 +1204,7 @@ main (int argc, char **argv) { fprintf (stderr, _("Both --no-clobber and --convert-links were specified," - "only --convert-links will be used.\n")); + " only --convert-links will be used.\n")); opt.noclobber = false; } @@ -1204,6 +1227,7 @@ main (int argc, char **argv) if (opt.verbose == -1) opt.verbose = !opt.quiet; + /* Sanity checks. */ if (opt.verbose && opt.quiet) { @@ -1299,7 +1323,7 @@ for details.\n\n")); } if (opt.warc_keep_log) { - opt.progress_type = "dot"; + opt.progress_type = xstrdup ("dot"); } } @@ -1323,6 +1347,93 @@ for details.\n\n")); exit (1); } + /* Compile the regular expressions. */ + switch (opt.regex_type) + { +#ifdef HAVE_LIBPCRE + case regex_type_pcre: + opt.regex_compile_fun = compile_pcre_regex; + opt.regex_match_fun = match_pcre_regex; + break; +#endif + + case regex_type_posix: + default: + opt.regex_compile_fun = compile_posix_regex; + opt.regex_match_fun = match_posix_regex; + break; + } + if (opt.acceptregex_s) + { + opt.acceptregex = opt.regex_compile_fun (opt.acceptregex_s); + if (!opt.acceptregex) + exit (1); + } + if (opt.rejectregex_s) + { + opt.rejectregex = opt.regex_compile_fun (opt.rejectregex_s); + if (!opt.rejectregex) + exit (1); + } + if (opt.post_data || opt.post_file_name) + { + if (opt.post_data && opt.post_file_name) + { + fprintf (stderr, _("You cannot specify both --post-data and --post-file.\n")); + exit (1); + } + else if (opt.method) + { + fprintf (stderr, _("You cannot use --post-data or --post-file along with --method. " + "--method expects data through --body-data and --body-file options")); + exit (1); + } + } + if (opt.body_data || opt.body_file) + { + if (!opt.method) + { + fprintf (stderr, _("You must specify a method through --method=HTTPMethod " + "to use with --body-data or --body-file.\n")); + exit (1); + } + else if (opt.body_data && opt.body_file) + { + fprintf (stderr, _("You cannot specify both --body-data and --body-file.\n")); + exit (1); + } + } + + /* Set various options as required for opt.method. */ + + /* When user specifies HEAD as the method, we do not wish to download any + files. Hence, set wget to run in spider mode. */ + if (opt.method && strcasecmp (opt.method, "HEAD") == 0) + setoptval ("spider", "1", "spider"); + + /* Convert post_data to body-data and post_file_name to body-file options. + This is required so as to remove redundant code later on in gethttp(). + The --post-data and --post-file options may also be removed in + the future hence it makes sense to convert them to aliases for + the more generic --method options. + This MUST occur only after the sanity checks so as to prevent the + user from setting both post and body options simultaneously. + */ + if (opt.post_data || opt.post_file_name) + { + setoptval ("method", "POST", "method"); + if (opt.post_data) + { + setoptval ("bodydata", opt.post_data, "body-data"); + opt.post_data = NULL; + } + else + { + setoptval ("bodyfile", opt.post_file_name, "body-file"); + opt.post_file_name = NULL; + } + } + #ifdef ENABLE_IRI if (opt.enable_iri) { @@ -1523,7 +1634,7 @@ outputting to a regular file.\n")); &dt, opt.recursive, iri, true); } - if (opt.delete_after && file_exists_p(filename)) + if (opt.delete_after && filename != NULL && file_exists_p (filename)) { DEBUGP (("Removing file due to --delete-after in main():\n")); logprintf (LOG_VERBOSE, _("Removing %s.\n"), filename); @@ -1590,14 +1701,6 @@ outputting to a regular file.\n")); if (opt.convert_links && !opt.delete_after) convert_all_links (); - /* Close WARC file. */ - if (opt.warc_filename != 0) - warc_close (); - - log_close (); - - for (i = 0; i < nurl; i++) - xfree (url[i]); cleanup (); exit (get_exit_status ());