X-Git-Url: http://sjero.net/git/?p=wget;a=blobdiff_plain;f=src%2Fmain.c;h=352715a0079187686bc7cf45649f05627bb459ee;hp=3a0525034aa09937b6b4f585ab0a2ee1f7b61c30;hb=408126aae0277e5c9e995a32bd942f4fa5cd7a9d;hpb=0db305126bd05f0739d83d36634772cf44851a65 diff --git a/src/main.c b/src/main.c index 3a052503..352715a0 100644 --- a/src/main.c +++ b/src/main.c @@ -55,13 +55,19 @@ as that of the covered work. */ #include "spider.h" #include "http.h" /* for save_cookies */ #include "ptimer.h" +#include "warc.h" #include #include #include +#ifdef WINDOWS +# include +# include +#endif + #ifdef __VMS -#include "vms.h" +# include "vms.h" #endif /* __VMS */ #ifndef PATH_SEPARATOR @@ -173,6 +179,7 @@ static struct cmdline_option option_data[] = { "continue", 'c', OPT_BOOLEAN, "continue", -1 }, { "convert-links", 'k', OPT_BOOLEAN, "convertlinks", -1 }, { "content-disposition", 0, OPT_BOOLEAN, "contentdisposition", -1 }, + { "content-on-error", 0, OPT_BOOLEAN, "contentonerror", -1 }, { "cookies", 0, OPT_BOOLEAN, "cookies", -1 }, { "cut-dirs", 0, OPT_VALUE, "cutdirs", -1 }, { WHEN_DEBUG ("debug"), 'd', OPT_BOOLEAN, "debug", -1 }, @@ -237,7 +244,7 @@ static struct cmdline_option option_data[] = { "post-data", 0, OPT_VALUE, "postdata", -1 }, { "post-file", 0, OPT_VALUE, "postfile", -1 }, { "prefer-family", 0, OPT_VALUE, "preferfamily", -1 }, - { "preserve-permissions", 0, OPT_BOOLEAN, "preservepermissions", -1 }, /* deprecated */ + { "preserve-permissions", 0, OPT_BOOLEAN, "preservepermissions", -1 }, { IF_SSL ("private-key"), 0, OPT_VALUE, "privatekey", -1 }, { IF_SSL ("private-key-type"), 0, OPT_VALUE, "privatekeytype", -1 }, { "progress", 0, OPT_VALUE, "progress", -1 }, @@ -281,6 +288,17 @@ static struct cmdline_option option_data[] = { "version", 'V', OPT_FUNCALL, (void *) print_version, no_argument }, { "wait", 'w', OPT_VALUE, "wait", -1 }, { "waitretry", 0, OPT_VALUE, "waitretry", -1 }, + { "warc-cdx", 0, OPT_BOOLEAN, "warccdx", -1 }, +#ifdef HAVE_LIBZ + { "warc-compression", 0, OPT_BOOLEAN, "warccompression", -1 }, +#endif + { "warc-dedup", 0, OPT_VALUE, "warccdxdedup", -1 }, + { "warc-digests", 0, OPT_BOOLEAN, "warcdigests", -1 }, + { "warc-file", 0, OPT_VALUE, "warcfile", -1 }, + { "warc-header", 0, OPT_VALUE, "warcheader", -1 }, + { "warc-keep-log", 0, OPT_BOOLEAN, "warckeeplog", -1 }, + { "warc-max-size", 0, OPT_VALUE, "warcmaxsize", -1 }, + { "warc-tempdir", 0, OPT_VALUE, "warctempdir", -1 }, #ifdef USE_WATT32 { "wdebug", 0, OPT_BOOLEAN, "wdebug", -1 }, #endif @@ -388,11 +406,11 @@ init_switches (void) } /* Print the usage message. */ -static void +static int print_usage (int error) { - fprintf (error ? stderr : stdout, _("Usage: %s [OPTION]... [URL]...\n"), - exec_name); + return fprintf (error ? stderr : stdout, + _("Usage: %s [OPTION]... [URL]...\n"), exec_name); } /* Print the help message, describing all the available options. If @@ -589,6 +607,8 @@ HTTP options:\n"), N_("\ --content-disposition honor the Content-Disposition header when\n\ choosing local file names (EXPERIMENTAL).\n"), + N_("\ + --content-on-error output the received content on server errors.\n"), N_("\ --auth-no-challenge send Basic HTTP authentication information\n\ without first waiting for the server's\n\ @@ -638,10 +658,37 @@ FTP options:\n"), --no-glob turn off FTP file name globbing.\n"), N_("\ --no-passive-ftp disable the \"passive\" transfer mode.\n"), + N_("\ + --preserve-permissions preserve remote file permissions.\n"), N_("\ --retr-symlinks when recursing, get linked-to files (not dir).\n"), "\n", + N_("\ +WARC options:\n"), + N_("\ + --warc-file=FILENAME save request/response data to a .warc.gz file.\n"), + N_("\ + --warc-header=STRING insert STRING into the warcinfo record.\n"), + N_("\ + --warc-max-size=NUMBER set maximum size of WARC files to NUMBER.\n"), + N_("\ + --warc-cdx write CDX index files.\n"), + N_("\ + --warc-dedup=FILENAME do not store records listed in this CDX file.\n"), +#ifdef HAVE_LIBZ + N_("\ + --no-warc-compression do not compress WARC files with GZIP.\n"), +#endif + N_("\ + --no-warc-digests do not calculate SHA1 digests.\n"), + N_("\ + --no-warc-keep-log do not store the log file in a WARC record.\n"), + N_("\ + --warc-tempdir=DIRECTORY location for temporary files created by the\n\ + WARC writer.\n"), + "\n", + N_("\ Recursive download:\n"), N_("\ @@ -704,12 +751,15 @@ Recursive accept/reject:\n"), size_t i; - printf (_("GNU Wget %s, a non-interactive network retriever.\n"), - version_string); - print_usage (0); + if (printf (_("GNU Wget %s, a non-interactive network retriever.\n"), + version_string) < 0) + exit (3); + if (print_usage (0) < 0) + exit (3); for (i = 0; i < countof (help); i++) - fputs (_(help[i]), stdout); + if (fputs (_(help[i]), stdout) < 0) + exit (3); exit (0); } @@ -754,7 +804,7 @@ prompt_for_password (void) to at most line_length. prefix is printed on the first line and an appropriate number of spaces are added on subsequent lines.*/ -static void +static int format_and_print_line (const char *prefix, const char *line, int line_length) { @@ -769,7 +819,8 @@ format_and_print_line (const char *prefix, const char *line, if (line_length <= 0) line_length = MAX_CHARS_PER_LINE - TABULATION; - printf ("%s", prefix); + if (printf ("%s", prefix) < 0) + return -1; remaining_chars = line_length; /* We break on spaces. */ token = strtok (line_dup, " "); @@ -780,17 +831,21 @@ format_and_print_line (const char *prefix, const char *line, token on the next line. */ if (remaining_chars <= strlen (token)) { - printf ("\n%*c", TABULATION, ' '); + if (printf ("\n%*c", TABULATION, ' ') < 0) + return -1; remaining_chars = line_length - TABULATION; } - printf ("%s ", token); + if (printf ("%s ", token) < 0) + return -1; remaining_chars -= strlen (token) + 1; /* account for " " */ token = strtok (NULL, " "); } - printf ("\n"); + if (printf ("\n") < 0) + return -1; xfree (line_dup); + return 0; } static void @@ -803,76 +858,96 @@ print_version (void) char *env_wgetrc, *user_wgetrc; int i; - printf (_("GNU Wget %s built on %s.\n\n"), version_string, OS_TYPE); + if (printf (_("GNU Wget %s built on %s.\n\n"), version_string, OS_TYPE) < 0) + exit (3); for (i = 0; compiled_features[i] != NULL; ) { int line_length = MAX_CHARS_PER_LINE; while ((line_length > 0) && (compiled_features[i] != NULL)) { - printf ("%s ", compiled_features[i]); + if (printf ("%s ", compiled_features[i]) < 0) + exit (3); line_length -= strlen (compiled_features[i]) + 2; i++; } - printf ("\n"); + if (printf ("\n") < 0) + exit (3); } - printf ("\n"); + if (printf ("\n") < 0) + exit (3); /* Handle the case when $WGETRC is unset and $HOME/.wgetrc is absent. */ - printf ("%s\n", wgetrc_title); + if (printf ("%s\n", wgetrc_title) < 0) + exit (3); + env_wgetrc = wgetrc_env_file_name (); if (env_wgetrc && *env_wgetrc) { - printf (_(" %s (env)\n"), env_wgetrc); + if (printf (_(" %s (env)\n"), env_wgetrc) < 0) + exit (3); xfree (env_wgetrc); } user_wgetrc = wgetrc_user_file_name (); if (user_wgetrc) { - printf (_(" %s (user)\n"), user_wgetrc); + if (printf (_(" %s (user)\n"), user_wgetrc) < 0) + exit (3); xfree (user_wgetrc); } #ifdef SYSTEM_WGETRC - printf (_(" %s (system)\n"), SYSTEM_WGETRC); + if (printf (_(" %s (system)\n"), SYSTEM_WGETRC) < 0) + exit (3); #endif #ifdef ENABLE_NLS - format_and_print_line (locale_title, + if (format_and_print_line (locale_title, LOCALEDIR, - MAX_CHARS_PER_LINE); + MAX_CHARS_PER_LINE) < 0) + exit (3); #endif /* def ENABLE_NLS */ if (compilation_string != NULL) - format_and_print_line (compile_title, - compilation_string, - MAX_CHARS_PER_LINE); + if (format_and_print_line (compile_title, + compilation_string, + MAX_CHARS_PER_LINE) < 0) + exit (3); if (link_string != NULL) - format_and_print_line (link_title, - link_string, - MAX_CHARS_PER_LINE); + if (format_and_print_line (link_title, + link_string, + MAX_CHARS_PER_LINE) < 0) + exit (3); + + if (printf ("\n") < 0) + exit (3); - printf ("\n"); /* TRANSLATORS: When available, an actual copyright character - (cirle-c) should be used in preference to "(C)". */ - fputs (_("\ -Copyright (C) 2009 Free Software Foundation, Inc.\n"), stdout); - fputs (_("\ + (circle-c) should be used in preference to "(C)". */ + if (fputs (_("\ +Copyright (C) 2011 Free Software Foundation, Inc.\n"), stdout) < 0) + exit (3); + if (fputs (_("\ License GPLv3+: GNU GPL version 3 or later\n\ .\n\ This is free software: you are free to change and redistribute it.\n\ -There is NO WARRANTY, to the extent permitted by law.\n"), stdout); +There is NO WARRANTY, to the extent permitted by law.\n"), stdout) < 0) + exit (3); /* TRANSLATORS: When available, please use the proper diacritics for names such as this one. See en_US.po for reference. */ - fputs (_("\nOriginally written by Hrvoje Niksic .\n"), - stdout); - fputs (_("Please send bug reports and questions to .\n"), - stdout); + if (fputs (_("\nOriginally written by Hrvoje Niksic .\n"), + stdout) < 0) + exit (3); + if (fputs (_("Please send bug reports and questions to .\n"), + stdout) < 0) + exit (3); + exit (0); } char *program_name; /* Needed by lib/error.c. */ +char *program_argstring; /* Needed by wget_warc.c. */ int main (int argc, char **argv) @@ -908,13 +983,34 @@ main (int argc, char **argv) windows_main ((char **) &exec_name); #endif + /* Construct the arguments string. */ + int argstring_length = 1; + for (i = 1; i < argc; i++) + argstring_length += strlen (argv[i]) + 2 + 1; + char *p = program_argstring = malloc (argstring_length * sizeof (char)); + if (p == NULL) + { + fprintf (stderr, _("Memory allocation problem\n")); + exit (2); + } + for (i = 1; i < argc; i++) + { + *p++ = '"'; + int arglen = strlen (argv[i]); + memcpy (p, argv[i], arglen); + p += arglen; + *p++ = '"'; + *p++ = ' '; + } + *p = '\0'; + /* Load the hard-coded defaults. */ defaults (); init_switches (); - /* This seperate getopt_long is needed to find the user config - and parse it before the other user options. */ + /* This separate getopt_long is needed to find the user config file + option ("--config") and parse it before the other user options. */ longindex = -1; int retconf; bool use_userconfig = false; @@ -925,20 +1021,25 @@ main (int argc, char **argv) int confval; bool userrc_ret = true; struct cmdline_option *config_opt; - confval = long_options[longindex].val; - config_opt = &option_data[confval & ~BOOLEAN_NEG_MARKER]; - if (strcmp (config_opt->long_name, "config") == 0) - { - userrc_ret &= run_wgetrc (optarg); - use_userconfig = true; - } - if (!userrc_ret) + + /* There is no short option for "--config". */ + if (longindex >= 0) { - printf ("Exiting due to error in %s\n", optarg); - exit (2); + confval = long_options[longindex].val; + config_opt = &option_data[confval & ~BOOLEAN_NEG_MARKER]; + if (strcmp (config_opt->long_name, "config") == 0) + { + userrc_ret &= run_wgetrc (optarg); + use_userconfig = true; + } + if (!userrc_ret) + { + fprintf (stderr, "Exiting due to error in %s\n", optarg); + exit (2); + } + else + break; } - else - break; } /* If the user did not specify a config, read the system wgetrc and ~/.wgetrc. */ @@ -961,9 +1062,10 @@ main (int argc, char **argv) { if (ret == '?') { - print_usage (0); - printf ("\n"); - printf (_("Try `%s --help' for more options.\n"), exec_name); + print_usage (1); + fprintf (stderr, "\n"); + fprintf (stderr, _("Try `%s --help' for more options.\n"), + exec_name); exit (2); } /* Find the short option character in the mapping. */ @@ -1152,6 +1254,47 @@ for details.\n\n")); } } + if (opt.warc_filename != 0) + { + if (opt.noclobber) + { + fprintf (stderr, + _("WARC output does not work with --no-clobber, " + "--no-clobber will be disabled.\n")); + opt.noclobber = false; + } + if (opt.timestamping) + { + fprintf (stderr, + _("WARC output does not work with timestamping, " + "timestamping will be disabled.\n")); + opt.timestamping = false; + } + if (opt.spider) + { + fprintf (stderr, + _("WARC output does not work with --spider.\n")); + exit (1); + } + if (opt.always_rest) + { + fprintf (stderr, + _("WARC output does not work with --continue, " + "--continue will be disabled.\n")); + opt.always_rest = false; + } + if (opt.warc_cdx_dedup_filename != 0 && !opt.warc_digests_enabled) + { + fprintf (stderr, + _("Digests are disabled; WARC deduplication will " + "not find duplicate records.\n")); + } + if (opt.warc_keep_log) + { + opt.progress_type = "dot"; + } + } + if (opt.ask_passwd && opt.passwd) { fprintf (stderr, @@ -1165,7 +1308,7 @@ for details.\n\n")); /* No URL specified. */ fprintf (stderr, _("%s: missing URL\n"), exec_name); print_usage (1); - printf ("\n"); + fprintf (stderr, "\n"); /* #### Something nicer should be printed here -- similar to the pre-1.5 `--help' page. */ fprintf (stderr, _("Try `%s --help' for more options.\n"), exec_name); @@ -1218,6 +1361,11 @@ for details.\n\n")); /* Fill in the arguments. */ url = alloca_array (char *, nurl + 1); + if (url == NULL) + { + fprintf (stderr, _("Memory allocation problem\n")); + exit (2); + } for (i = 0; i < nurl; i++, optind++) { char *rewritten = rewrite_shorthand_url (argv[optind]); @@ -1231,6 +1379,10 @@ for details.\n\n")); /* Initialize logging. */ log_init (opt.lfilename, append_to_log); + /* Open WARC file. */ + if (opt.warc_filename != 0) + warc_init (); + DEBUGP (("DEBUG output created by Wget %s on %s.\n\n", version_string, OS_TYPE)); @@ -1249,14 +1401,7 @@ for details.\n\n")); if (HYPHENP (opt.output_document)) { #ifdef WINDOWS - FILE *result; - result = freopen ("CONOUT$", "wb", stdout); - if (result == NULL) - { - logputs (LOG_NOTQUIET, _("\ -WARNING: Can't reopen standard output in binary mode;\n\ - downloaded file may contain inappropriate line endings.\n")); - } + _setmode (_fileno (stdout), _O_BINARY); #endif output_stream = stdout; } @@ -1437,12 +1582,17 @@ outputting to a regular file.\n")); if (opt.convert_links && !opt.delete_after) convert_all_links (); + /* Close WARC file. */ + if (opt.warc_filename != 0) + warc_close (); + log_close (); + for (i = 0; i < nurl; i++) xfree (url[i]); cleanup (); - return get_exit_status (); + exit (get_exit_status ()); } #endif /* TESTING */