X-Git-Url: http://sjero.net/git/?p=wget;a=blobdiff_plain;f=src%2Finit.c;h=fbef133cf9d295217b1a166b996ccb5cfaf67242;hp=5ab0862cbcf527c729fbff534382608c817f4450;hb=38a7829dcb4eb5dba28dbf0f05c6a80fea9217f8;hpb=78a765739543d5ca3bc83ad70f5d391b9d2af4f5 diff --git a/src/init.c b/src/init.c index 5ab0862c..fbef133c 100644 --- a/src/init.c +++ b/src/init.c @@ -1,6 +1,7 @@ /* Reading/parsing the initialization file. - Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, - 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc. + Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, + 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 Free Software Foundation, + Inc. This file is part of GNU Wget. @@ -29,14 +30,27 @@ shall include the source code for the parts of OpenSSL used as well as that of the covered work. */ #include "wget.h" +#include "exits.h" #include #include -#ifdef HAVE_UNISTD_H -# include -#endif +#include +#include #include #include +#include +/* not all systems provide PATH_MAX in limits.h */ +#ifndef PATH_MAX +# include +# ifndef PATH_MAX +# define PATH_MAX MAXPATHLEN +# endif +#endif + +#include +#ifdef HAVE_LIBPCRE +# include +#endif #ifdef HAVE_PWD_H # include @@ -53,16 +67,13 @@ as that of the covered work. */ #include "res.h" /* for res_cleanup */ #include "http.h" /* for http_cleanup */ #include "retr.h" /* for output_stream */ +#include "warc.h" /* for warc_close */ +#include "spider.h" /* for spider_cleanup */ #ifdef TESTING #include "test.h" #endif -/* We want tilde expansion enabled only when reading `.wgetrc' lines; - otherwise, it will be performed by the shell. This variable will - be set by the wgetrc-reading function. */ - -static bool enable_tilde_expansion; #define CMD_DECLARE(func) static bool func (const char *, const char *, void *) @@ -77,6 +88,7 @@ CMD_DECLARE (cmd_directory_vector); CMD_DECLARE (cmd_number); CMD_DECLARE (cmd_number_inf); CMD_DECLARE (cmd_string); +CMD_DECLARE (cmd_string_uppercase); CMD_DECLARE (cmd_file); CMD_DECLARE (cmd_directory); CMD_DECLARE (cmd_time); @@ -84,12 +96,15 @@ CMD_DECLARE (cmd_vector); CMD_DECLARE (cmd_spec_dirstruct); CMD_DECLARE (cmd_spec_header); +CMD_DECLARE (cmd_spec_warc_header); CMD_DECLARE (cmd_spec_htmlify); CMD_DECLARE (cmd_spec_mirror); CMD_DECLARE (cmd_spec_prefer_family); CMD_DECLARE (cmd_spec_progress); CMD_DECLARE (cmd_spec_recursive); +CMD_DECLARE (cmd_spec_regex_type); CMD_DECLARE (cmd_spec_restrict_file_names); +CMD_DECLARE (cmd_spec_report_speed); #ifdef HAVE_SSL CMD_DECLARE (cmd_spec_secure_protocol); #endif @@ -111,7 +126,9 @@ static const struct { } commands[] = { /* KEEP THIS LIST ALPHABETICALLY SORTED */ { "accept", &opt.accepts, cmd_vector }, + { "acceptregex", &opt.acceptregex_s, cmd_string }, { "addhostdir", &opt.add_hostdir, cmd_boolean }, + { "adjustextension", &opt.adjust_extension, cmd_boolean }, { "alwaysrest", &opt.always_rest, cmd_boolean }, /* deprecated */ { "askpassword", &opt.ask_passwd, cmd_boolean }, { "authnochallenge", &opt.auth_without_challenge, @@ -121,6 +138,8 @@ static const struct { { "backups", &opt.backups, cmd_number }, { "base", &opt.base_href, cmd_string }, { "bindaddress", &opt.bind_address, cmd_string }, + { "bodydata", &opt.body_data, cmd_string }, + { "bodyfile", &opt.body_file, cmd_string }, #ifdef HAVE_SSL { "cacertificate", &opt.ca_cert, cmd_file }, #endif @@ -131,16 +150,16 @@ static const struct { { "certificatetype", &opt.cert_type, cmd_cert_type }, { "checkcertificate", &opt.check_cert, cmd_boolean }, #endif + { "chooseconfig", &opt.choose_config, cmd_file }, { "connecttimeout", &opt.connect_timeout, cmd_time }, { "contentdisposition", &opt.content_disposition, cmd_boolean }, + { "contentonerror", &opt.content_on_error, cmd_boolean }, { "continue", &opt.always_rest, cmd_boolean }, { "convertlinks", &opt.convert_links, cmd_boolean }, { "cookies", &opt.cookies, cmd_boolean }, { "cutdirs", &opt.cut_dirs, cmd_number }, -#ifdef ENABLE_DEBUG { "debug", &opt.debug, cmd_boolean }, -#endif - { "defaultpage", &opt.default_page, cmd_string}, + { "defaultpage", &opt.default_page, cmd_string }, { "deleteafter", &opt.delete_after, cmd_boolean }, { "dirprefix", &opt.dir_prefix, cmd_directory }, { "dirstruct", NULL, cmd_spec_dirstruct }, @@ -150,7 +169,7 @@ static const struct { { "dotbytes", &opt.dot_bytes, cmd_bytes }, { "dotsinline", &opt.dots_in_line, cmd_number }, { "dotspacing", &opt.dot_spacing, cmd_number }, - { "dotstyle", &opt.dot_style, cmd_string }, + { "dotstyle", &opt.dot_style, cmd_string }, /* deprecated */ #ifdef HAVE_SSL { "egdfile", &opt.egd_file, cmd_file }, #endif @@ -162,15 +181,21 @@ static const struct { { "ftppasswd", &opt.ftp_passwd, cmd_string }, /* deprecated */ { "ftppassword", &opt.ftp_passwd, cmd_string }, { "ftpproxy", &opt.ftp_proxy, cmd_string }, +#ifdef __VMS + { "ftpstmlf", &opt.ftp_stmlf, cmd_boolean }, +#endif /* def __VMS */ { "ftpuser", &opt.ftp_user, cmd_string }, { "glob", &opt.ftp_glob, cmd_boolean }, { "header", NULL, cmd_spec_header }, - { "htmlextension", &opt.html_extension, cmd_boolean }, + { "htmlextension", &opt.adjust_extension, cmd_boolean }, /* deprecated */ { "htmlify", NULL, cmd_spec_htmlify }, { "httpkeepalive", &opt.http_keep_alive, cmd_boolean }, { "httppasswd", &opt.http_passwd, cmd_string }, /* deprecated */ { "httppassword", &opt.http_passwd, cmd_string }, { "httpproxy", &opt.http_proxy, cmd_string }, +#ifdef HAVE_SSL + { "httpsonly", &opt.https_only, cmd_boolean }, +#endif { "httpsproxy", &opt.https_proxy, cmd_string }, { "httpuser", &opt.http_user, cmd_string }, { "ignorecase", &opt.ignore_case, cmd_boolean }, @@ -186,13 +211,15 @@ static const struct { { "keepsessioncookies", &opt.keep_session_cookies, cmd_boolean }, { "limitrate", &opt.limit_rate, cmd_bytes }, { "loadcookies", &opt.cookies_input, cmd_file }, - { "locale", &opt.locale, cmd_string }, + { "localencoding", &opt.locale, cmd_string }, { "logfile", &opt.lfilename, cmd_file }, { "login", &opt.ftp_user, cmd_string },/* deprecated*/ { "maxredirect", &opt.max_redirect, cmd_number }, + { "method", &opt.method, cmd_string_uppercase }, { "mirror", NULL, cmd_spec_mirror }, { "netrc", &opt.netrc, cmd_boolean }, { "noclobber", &opt.noclobber, cmd_boolean }, + { "noconfig", &opt.noconfig, cmd_boolean }, { "noparent", &opt.no_parent, cmd_boolean }, { "noproxy", &opt.no_proxy, cmd_vector }, { "numtries", &opt.ntry, cmd_number_inf },/* deprecated*/ @@ -224,10 +251,13 @@ static const struct { { "reclevel", &opt.reclevel, cmd_number_inf }, { "recursive", NULL, cmd_spec_recursive }, { "referer", &opt.referer, cmd_string }, + { "regextype", &opt.regex_type, cmd_spec_regex_type }, { "reject", &opt.rejects, cmd_vector }, + { "rejectregex", &opt.rejectregex_s, cmd_string }, { "relativeonly", &opt.relative_only, cmd_boolean }, { "remoteencoding", &opt.encoding_remote, cmd_string }, { "removelisting", &opt.remove_listing, cmd_boolean }, + { "reportspeed", &opt.report_bps, cmd_spec_report_speed}, { "restrictfilenames", NULL, cmd_spec_restrict_file_names }, { "retrsymlinks", &opt.retr_symlinks, cmd_boolean }, { "retryconnrefused", &opt.retry_connrefused, cmd_boolean }, @@ -238,19 +268,36 @@ static const struct { { "secureprotocol", &opt.secure_protocol, cmd_spec_secure_protocol }, #endif { "serverresponse", &opt.server_response, cmd_boolean }, + { "showalldnsentries", &opt.show_all_dns_entries, cmd_boolean }, + { "showprogress", &opt.show_progress, cmd_boolean }, { "spanhosts", &opt.spanhost, cmd_boolean }, { "spider", &opt.spider, cmd_boolean }, + { "startpos", &opt.start_pos, cmd_bytes }, { "strictcomments", &opt.strict_comments, cmd_boolean }, { "timeout", NULL, cmd_spec_timeout }, { "timestamping", &opt.timestamping, cmd_boolean }, { "tries", &opt.ntry, cmd_number_inf }, + { "trustservernames", &opt.trustservernames, cmd_boolean }, + { "unlink", &opt.unlink, cmd_boolean }, { "useproxy", &opt.use_proxy, cmd_boolean }, { "user", &opt.user, cmd_string }, { "useragent", NULL, cmd_spec_useragent }, + { "useservertimestamps", &opt.useservertimestamps, cmd_boolean }, { "verbose", NULL, cmd_spec_verbose }, { "wait", &opt.wait, cmd_time }, { "waitretry", &opt.waitretry, cmd_time }, -#ifdef MSDOS + { "warccdx", &opt.warc_cdx_enabled, cmd_boolean }, + { "warccdxdedup", &opt.warc_cdx_dedup_filename, cmd_file }, +#ifdef HAVE_LIBZ + { "warccompression", &opt.warc_compression_enabled, cmd_boolean }, +#endif + { "warcdigests", &opt.warc_digests_enabled, cmd_boolean }, + { "warcfile", &opt.warc_filename, cmd_file }, + { "warcheader", NULL, cmd_spec_warc_header }, + { "warckeeplog", &opt.warc_keep_log, cmd_boolean }, + { "warcmaxsize", &opt.warc_maxsize, cmd_bytes }, + { "warctempdir", &opt.warc_tempdir, cmd_directory }, +#ifdef USE_WATT32 { "wdebug", &opt.wdebug, cmd_boolean }, #endif }; @@ -280,7 +327,7 @@ command_by_name (const char *cmdname) } /* Reset the variables to default values. */ -static void +void defaults (void) { char *tmp; @@ -331,8 +378,11 @@ defaults (void) opt.restrict_files_os = restrict_unix; #endif opt.restrict_files_ctrl = true; + opt.restrict_files_nonascii = false; opt.restrict_files_case = restrict_no_case_restriction; + opt.regex_type = regex_type_posix; + opt.max_redirect = 20; opt.waitretry = 10; @@ -344,6 +394,25 @@ defaults (void) #endif opt.locale = NULL; opt.encoding_remote = NULL; + + opt.useservertimestamps = true; + opt.show_all_dns_entries = false; + + opt.warc_maxsize = 0; /* 1024 * 1024 * 1024; */ +#ifdef HAVE_LIBZ + opt.warc_compression_enabled = true; +#else + opt.warc_compression_enabled = false; +#endif + opt.warc_digests_enabled = true; + opt.warc_cdx_enabled = false; + opt.warc_cdx_dedup_filename = NULL; + opt.warc_tempdir = NULL; + opt.warc_keep_log = true; + + /* Use a negative value to mark the absence of --start-pos option */ + opt.start_pos = -1; + opt.show_progress = false; } /* Return the user's home directory (strdup-ed), or NULL if none is @@ -351,8 +420,8 @@ defaults (void) char * home_dir (void) { - static char buf[PATH_MAX]; - static char *home; + static char *buf = NULL; + static char *home, *ret; if (!home) { @@ -360,17 +429,28 @@ home_dir (void) if (!home) { #if defined(MSDOS) + int len; + /* Under MSDOS, if $HOME isn't defined, use the directory where `wget.exe' resides. */ const char *_w32_get_argv0 (void); /* in libwatt.a/pcconfig.c */ char *p; - strcpy (buf, _w32_get_argv0 ()); + buff = _w32_get_argv0 (); + p = strrchr (buf, '/'); /* djgpp */ if (!p) p = strrchr (buf, '\\'); /* others */ assert (p); - *p = '\0'; + + len = p - buff + 1; + buff = malloc (len + 1); + if (buff == NULL) + return NULL; + + strncpy (buff, _w32_get_argv0 (), len); + buff[len] = '\0'; + home = buf; #elif !defined(WINDOWS) /* If HOME is not defined, try getting it from the password @@ -378,8 +458,7 @@ home_dir (void) struct passwd *pwd = getpwuid (getuid ()); if (!pwd || !pwd->pw_dir) return NULL; - strcpy (buf, pwd->pw_dir); - home = buf; + home = pwd->pw_dir; #else /* !WINDOWS */ /* Under Windows, if $HOME isn't defined, use the directory where `wget.exe' resides. */ @@ -388,15 +467,18 @@ home_dir (void) } } - return home ? xstrdup (home) : NULL; + ret = home ? xstrdup (home) : NULL; + free (buf); + + return ret; } -/* Check the 'WGETRC' environment variable and return the file name - if 'WGETRC' is set and is a valid file. +/* Check the 'WGETRC' environment variable and return the file name + if 'WGETRC' is set and is a valid file. If the `WGETRC' variable exists but the file does not exist, the function will exit(). */ char * -wgetrc_env_file_name (void) +wgetrc_env_file_name (void) { char *env = getenv ("WGETRC"); if (env && *env) @@ -412,16 +494,24 @@ wgetrc_env_file_name (void) return NULL; } -/* Check for the existance of '$HOME/.wgetrc' and return it's path +/* Check for the existance of '$HOME/.wgetrc' and return its path if it exists and is set. */ char * -wgetrc_user_file_name (void) +wgetrc_user_file_name (void) { - char *home = home_dir (); + char *home; char *file = NULL; + /* If that failed, try $HOME/.wgetrc (or equivalent). */ + +#ifdef __VMS + file = "SYS$LOGIN:.wgetrc"; +#else /* def __VMS */ + home = home_dir (); if (home) file = aprintf ("%s/.wgetrc", home); xfree_null (home); +#endif /* def __VMS [else] */ + if (!file) return NULL; if (!file_exists_p (file)) @@ -435,16 +525,15 @@ wgetrc_user_file_name (void) /* Return the path to the user's .wgetrc. This is either the value of `WGETRC' environment variable, or `$HOME/.wgetrc'. - Additionally, for windows, look in the directory where wget.exe + Additionally, for windows, look in the directory where wget.exe resides. */ char * wgetrc_file_name (void) { - char *home = NULL; char *file = wgetrc_env_file_name (); if (file && *file) return file; - + file = wgetrc_user_file_name (); #ifdef WINDOWS @@ -452,25 +541,25 @@ wgetrc_file_name (void) `wget.ini' in the directory where `wget.exe' resides; we do this for backward compatibility with previous versions of Wget. SYSTEM_WGETRC should not be defined under WINDOWS. */ - home = home_dir (); - if (!file || !file_exists_p (file)) + if (!file) { + char *home = home_dir (); xfree_null (file); file = NULL; home = ws_mypath (); if (home) - file = aprintf ("%s/wget.ini", home); + { + file = aprintf ("%s/wget.ini", home); + if (!file_exists_p (file)) + { + xfree (file); + file = NULL; + } + xfree (home); + } } - xfree_null (home); #endif /* WINDOWS */ - if (!file) - return NULL; - if (!file_exists_p (file)) - { - xfree (file); - return NULL; - } return file; } @@ -484,28 +573,29 @@ enum parse_line { static enum parse_line parse_line (const char *, char **, char **, int *); static bool setval_internal (int, const char *, const char *); +static bool setval_internal_tilde (int, const char *, const char *); /* Initialize variables from a wgetrc file. Returns zero (failure) if there were errors in the file. */ -static bool +bool run_wgetrc (const char *file) { FILE *fp; - char *line; + char *line = NULL; + size_t bufsize = 0; int ln; int errcnt = 0; - fp = fopen (file, "rb"); + fp = fopen (file, "r"); if (!fp) { fprintf (stderr, _("%s: Cannot read %s (%s).\n"), exec_name, file, strerror (errno)); return true; /* not a fatal error */ } - enable_tilde_expansion = true; ln = 1; - while ((line = read_whole_line (fp)) != NULL) + while (getline (&line, &bufsize, fp) > 0) { char *com = NULL, *val = NULL; int comind; @@ -515,7 +605,7 @@ run_wgetrc (const char *file) { case line_ok: /* If everything is OK, set the value. */ - if (!setval_internal (comind, com, val)) + if (!setval_internal_tilde (comind, com, val)) { fprintf (stderr, _("%s: Error in %s at line %d.\n"), exec_name, file, ln); @@ -539,10 +629,9 @@ run_wgetrc (const char *file) } xfree_null (com); xfree_null (val); - xfree (line); ++ln; } - enable_tilde_expansion = false; + xfree (line); fclose (fp); return errcnt == 0; @@ -553,16 +642,40 @@ run_wgetrc (const char *file) void initialize (void) { - char *file; - int ok = true; + char *file, *env_sysrc; + bool ok = true; - /* Load the hard-coded defaults. */ - defaults (); - - /* If SYSTEM_WGETRC is defined, use it. */ + /* Run a non-standard system rc file when the according environment + variable has been set. For internal testing purposes only! */ + env_sysrc = getenv ("SYSTEM_WGETRC"); + if (env_sysrc && file_exists_p (env_sysrc)) + { + ok &= run_wgetrc (env_sysrc); + /* If there are any problems parsing the system wgetrc file, tell + the user and exit */ + if (! ok) + { + fprintf (stderr, _("\ +Parsing system wgetrc file (env SYSTEM_WGETRC) failed. Please check\n\ +'%s',\n\ +or specify a different file using --config.\n"), env_sysrc); + exit (2); + } + } + /* Otherwise, if SYSTEM_WGETRC is defined, use it. */ #ifdef SYSTEM_WGETRC - if (file_exists_p (SYSTEM_WGETRC)) + else if (file_exists_p (SYSTEM_WGETRC)) ok &= run_wgetrc (SYSTEM_WGETRC); + /* If there are any problems parsing the system wgetrc file, tell + the user and exit */ + if (! ok) + { + fprintf (stderr, _("\ +Parsing system wgetrc file failed. Please check\n\ +'%s',\n\ +or specify a different file using --config.\n"), SYSTEM_WGETRC); + exit (2); + } #endif /* Override it with your own, if one exists. */ file = wgetrc_file_name (); @@ -674,6 +787,12 @@ parse_line (const char *line, char **com, char **val, int *comind) return line_ok; } +#if defined(WINDOWS) || defined(MSDOS) +# define ISSEP(c) ((c) == '/' || (c) == '\\') +#else +# define ISSEP(c) ((c) == '/') +#endif + /* Run commands[comind].action. */ static bool @@ -684,6 +803,37 @@ setval_internal (int comind, const char *com, const char *val) return commands[comind].action (com, val, commands[comind].place); } +static bool +setval_internal_tilde (int comind, const char *com, const char *val) +{ + bool ret; + int homelen; + char *home; + char **pstring; + ret = setval_internal (comind, com, val); + + /* We make tilde expansion for cmd_file and cmd_directory */ + if (((commands[comind].action == cmd_file) || + (commands[comind].action == cmd_directory)) + && ret && (*val == '~' && ISSEP (val[1]))) + { + pstring = commands[comind].place; + home = home_dir (); + if (home) + { + homelen = strlen (home); + while (homelen && ISSEP (home[homelen - 1])) + home[--homelen] = '\0'; + + /* Skip the leading "~/". */ + for (++val; ISSEP (*val); val++) + ; + *pstring = concat_strings (home, "/", val, (char *)0); + } + } + return ret; +} + /* Run command COM with value VAL. If running the command produces an error, report the error and exit. @@ -713,11 +863,11 @@ setoptval (const char *com, const char *val, const char *optname) This is used by the `--execute' flag in main.c. */ void -run_command (const char *opt) +run_command (const char *cmdopt) { char *com, *val; int comind; - switch (parse_line (opt, &com, &val, &comind)) + switch (parse_line (cmdopt, &com, &val, &comind)) { case line_ok: if (!setval_internal (comind, com, val)) @@ -727,7 +877,7 @@ run_command (const char *opt) break; default: fprintf (stderr, _("%s: Invalid --execute command %s\n"), - exec_name, quote (opt)); + exec_name, quote (cmdopt)); exit (2); } } @@ -810,7 +960,7 @@ cmd_number_inf (const char *com, const char *val, void *place) /* Copy (strdup) the string at COM to a new location and place a pointer to *PLACE. */ static bool -cmd_string (const char *com, const char *val, void *place) +cmd_string (const char *com _GL_UNUSED, const char *val, void *place) { char **pstring = (char **)place; @@ -819,17 +969,29 @@ cmd_string (const char *com, const char *val, void *place) return true; } -#if defined(WINDOWS) || defined(MSDOS) -# define ISSEP(c) ((c) == '/' || (c) == '\\') -#else -# define ISSEP(c) ((c) == '/') -#endif +/* Like cmd_string but ensure the string is upper case. */ +static bool +cmd_string_uppercase (const char *com _GL_UNUSED, const char *val, void *place) +{ + char *q, **pstring; + pstring = (char **)place; + xfree_null (*pstring); + + *pstring = xmalloc (strlen (val) + 1); + + for (q = *pstring; *val; val++, q++) + *q = c_toupper (*val); + + *q = '\0'; + return true; +} -/* Like the above, but handles tilde-expansion when reading a user's + +/* Like cmd_string, but handles tilde-expansion when reading a user's `.wgetrc'. In that case, and if VAL begins with `~', the tilde gets expanded to the user's home directory. */ static bool -cmd_file (const char *com, const char *val, void *place) +cmd_file (const char *com _GL_UNUSED, const char *val, void *place) { char **pstring = (char **)place; @@ -837,28 +999,7 @@ cmd_file (const char *com, const char *val, void *place) /* #### If VAL is empty, perhaps should set *PLACE to NULL. */ - if (!enable_tilde_expansion || !(*val == '~' && ISSEP (val[1]))) - { - noexpand: - *pstring = xstrdup (val); - } - else - { - int homelen; - char *home = home_dir (); - if (!home) - goto noexpand; - - homelen = strlen (home); - while (homelen && ISSEP (home[homelen - 1])) - home[--homelen] = '\0'; - - /* Skip the leading "~/". */ - for (++val; ISSEP (*val); val++) - ; - - *pstring = concat_strings (home, "/", val, (char *) 0); - } + *pstring = xstrdup (val); #if defined(WINDOWS) || defined(MSDOS) /* Convert "\" to "/". */ @@ -897,7 +1038,7 @@ cmd_directory (const char *com, const char *val, void *place) PLACE vector is cleared instead. */ static bool -cmd_vector (const char *com, const char *val, void *place) +cmd_vector (const char *com _GL_UNUSED, const char *val, void *place) { char ***pvec = (char ***)place; @@ -912,7 +1053,7 @@ cmd_vector (const char *com, const char *val, void *place) } static bool -cmd_directory_vector (const char *com, const char *val, void *place) +cmd_directory_vector (const char *com _GL_UNUSED, const char *val, void *place) { char ***pvec = (char ***)place; @@ -1128,7 +1269,7 @@ cmd_cert_type (const char *com, const char *val, void *place) static bool check_user_specified_header (const char *); static bool -cmd_spec_dirstruct (const char *com, const char *val, void *place_ignored) +cmd_spec_dirstruct (const char *com, const char *val, void *place_ignored _GL_UNUSED) { if (!cmd_boolean (com, val, &opt.dirstruct)) return false; @@ -1142,7 +1283,7 @@ cmd_spec_dirstruct (const char *com, const char *val, void *place_ignored) } static bool -cmd_spec_header (const char *com, const char *val, void *place_ignored) +cmd_spec_header (const char *com, const char *val, void *place_ignored _GL_UNUSED) { /* Empty value means reset the list of headers. */ if (*val == '\0') @@ -1163,7 +1304,28 @@ cmd_spec_header (const char *com, const char *val, void *place_ignored) } static bool -cmd_spec_htmlify (const char *com, const char *val, void *place_ignored) +cmd_spec_warc_header (const char *com, const char *val, void *place_ignored _GL_UNUSED) +{ + /* Empty value means reset the list of headers. */ + if (*val == '\0') + { + free_vec (opt.warc_user_headers); + opt.warc_user_headers = NULL; + return true; + } + + if (!check_user_specified_header (val)) + { + fprintf (stderr, _("%s: %s: Invalid WARC header %s.\n"), + exec_name, com, quote (val)); + return false; + } + opt.warc_user_headers = vec_append (opt.warc_user_headers, val); + return true; +} + +static bool +cmd_spec_htmlify (const char *com, const char *val, void *place_ignored _GL_UNUSED) { int flag = cmd_boolean (com, val, &opt.htmlify); if (flag && !opt.htmlify) @@ -1175,7 +1337,7 @@ cmd_spec_htmlify (const char *com, const char *val, void *place_ignored) no limit on max. recursion depth, and don't remove listings. */ static bool -cmd_spec_mirror (const char *com, const char *val, void *place_ignored) +cmd_spec_mirror (const char *com, const char *val, void *place_ignored _GL_UNUSED) { int mirror; @@ -1197,7 +1359,7 @@ cmd_spec_mirror (const char *com, const char *val, void *place_ignored) "IPv4", "IPv6", and "none". */ static bool -cmd_spec_prefer_family (const char *com, const char *val, void *place_ignored) +cmd_spec_prefer_family (const char *com, const char *val, void *place_ignored _GL_UNUSED) { static const struct decode_item choices[] = { { "IPv4", prefer_ipv4 }, @@ -1216,7 +1378,7 @@ cmd_spec_prefer_family (const char *com, const char *val, void *place_ignored) implementation before that. */ static bool -cmd_spec_progress (const char *com, const char *val, void *place_ignored) +cmd_spec_progress (const char *com, const char *val, void *place_ignored _GL_UNUSED) { if (!valid_progress_implementation_p (val)) { @@ -1237,7 +1399,7 @@ cmd_spec_progress (const char *com, const char *val, void *place_ignored) is specified. */ static bool -cmd_spec_recursive (const char *com, const char *val, void *place_ignored) +cmd_spec_recursive (const char *com, const char *val, void *place_ignored _GL_UNUSED) { if (!cmd_boolean (com, val, &opt.recursive)) return false; @@ -1249,12 +1411,32 @@ cmd_spec_recursive (const char *com, const char *val, void *place_ignored) return true; } +/* Validate --regex-type and set the choice. */ + +static bool +cmd_spec_regex_type (const char *com, const char *val, void *place_ignored _GL_UNUSED) +{ + static const struct decode_item choices[] = { + { "posix", regex_type_posix }, +#ifdef HAVE_LIBPCRE + { "pcre", regex_type_pcre }, +#endif + }; + int regex_type = regex_type_posix; + int ok = decode_string (val, choices, countof (choices), ®ex_type); + if (!ok) + fprintf (stderr, _("%s: %s: Invalid value %s.\n"), exec_name, com, quote (val)); + opt.regex_type = regex_type; + return ok; +} + static bool -cmd_spec_restrict_file_names (const char *com, const char *val, void *place_ignored) +cmd_spec_restrict_file_names (const char *com, const char *val, void *place_ignored _GL_UNUSED) { int restrict_os = opt.restrict_files_os; int restrict_ctrl = opt.restrict_files_ctrl; int restrict_case = opt.restrict_files_case; + int restrict_nonascii = opt.restrict_files_nonascii; const char *end; @@ -1265,7 +1447,7 @@ cmd_spec_restrict_file_names (const char *com, const char *val, void *place_igno end = strchr (val, ','); if (!end) end = val + strlen (val); - + if (VAL_IS ("unix")) restrict_os = restrict_unix; else if (VAL_IS ("windows")) @@ -1276,15 +1458,18 @@ cmd_spec_restrict_file_names (const char *com, const char *val, void *place_igno restrict_case = restrict_uppercase; else if (VAL_IS ("nocontrol")) restrict_ctrl = false; + else if (VAL_IS ("ascii")) + restrict_nonascii = true; else { - fprintf (stderr, - _("%s: %s: Invalid restriction %s, use [unix|windows],[lowercase|uppercase],[nocontrol].\n"), + fprintf (stderr, _("\ +%s: %s: Invalid restriction %s,\n\ + use [unix|windows],[lowercase|uppercase],[nocontrol],[ascii].\n"), exec_name, com, quote (val)); return false; } - if (*end) + if (*end) val = end + 1; } while (*val && *end); @@ -1294,10 +1479,20 @@ cmd_spec_restrict_file_names (const char *com, const char *val, void *place_igno opt.restrict_files_os = restrict_os; opt.restrict_files_ctrl = restrict_ctrl; opt.restrict_files_case = restrict_case; - + opt.restrict_files_nonascii = restrict_nonascii; + return true; } +static bool +cmd_spec_report_speed (const char *com, const char *val, void *place_ignored _GL_UNUSED) +{ + opt.report_bps = strcasecmp (val, "bits") == 0; + if (!opt.report_bps) + fprintf (stderr, _("%s: %s: Invalid value %s.\n"), exec_name, com, quote (val)); + return opt.report_bps; +} + #ifdef HAVE_SSL static bool cmd_spec_secure_protocol (const char *com, const char *val, void *place) @@ -1307,6 +1502,7 @@ cmd_spec_secure_protocol (const char *com, const char *val, void *place) { "sslv2", secure_protocol_sslv2 }, { "sslv3", secure_protocol_sslv3 }, { "tlsv1", secure_protocol_tlsv1 }, + { "pfs", secure_protocol_pfs }, }; int ok = decode_string (val, choices, countof (choices), place); if (!ok) @@ -1318,7 +1514,7 @@ cmd_spec_secure_protocol (const char *com, const char *val, void *place) /* Set all three timeout values. */ static bool -cmd_spec_timeout (const char *com, const char *val, void *place_ignored) +cmd_spec_timeout (const char *com, const char *val, void *place_ignored _GL_UNUSED) { double value; if (!cmd_time (com, val, &value)) @@ -1330,7 +1526,7 @@ cmd_spec_timeout (const char *com, const char *val, void *place_ignored) } static bool -cmd_spec_useragent (const char *com, const char *val, void *place_ignored) +cmd_spec_useragent (const char *com, const char *val, void *place_ignored _GL_UNUSED) { /* Disallow embedded newlines. */ if (strchr (val, '\n')) @@ -1349,7 +1545,7 @@ cmd_spec_useragent (const char *com, const char *val, void *place_ignored) some random hackery for disallowing -q -v). */ static bool -cmd_spec_verbose (const char *com, const char *val, void *place_ignored) +cmd_spec_verbose (const char *com, const char *val, void *place_ignored _GL_UNUSED) { bool flag; if (cmd_boolean (com, val, &flag)) @@ -1502,18 +1698,22 @@ decode_string (const char *val, const struct decode_item *items, int itemcount, return false; } - -void cleanup_html_url (void); - - /* Free the memory allocated by global variables. */ void cleanup (void) { /* Free external resources, close files, etc. */ + /* Close WARC file. */ + if (opt.warc_filename != 0) + warc_close (); + + log_close (); + if (output_stream) - fclose (output_stream); + if (fclose (output_stream) == EOF) + inform_exit_status (CLOSEFAILED); + /* No need to check for error because Wget flushes its output (and checks for errors) after any data arrives. */ @@ -1530,13 +1730,15 @@ cleanup (void) res_cleanup (); http_cleanup (); cleanup_html_url (); + spider_cleanup (); host_cleanup (); log_cleanup (); + netrc_cleanup (netrc_list); - { - extern acc_t *netrc_list; - free_netrc (netrc_list); - } + for (i = 0; i < nurl; i++) + xfree (url[i]); + + xfree_null (opt.choose_config); xfree_null (opt.lfilename); xfree_null (opt.dir_prefix); xfree_null (opt.input_filename); @@ -1560,6 +1762,7 @@ cleanup (void) xfree_null (opt.http_user); xfree_null (opt.http_passwd); free_vec (opt.user_headers); + free_vec (opt.warc_user_headers); # ifdef HAVE_SSL xfree_null (opt.cert_file); xfree_null (opt.private_key); @@ -1574,7 +1777,8 @@ cleanup (void) xfree_null (opt.user); xfree_null (opt.passwd); xfree_null (opt.base_href); - + xfree_null (opt.method); + #endif /* DEBUG_MALLOC */ } @@ -1583,34 +1787,27 @@ cleanup (void) #ifdef TESTING const char * -test_commands_sorted() +test_commands_sorted(void) { - int prev_idx = 0, next_idx = 1; - int command_count = countof (commands) - 1; - int cmp = 0; - while (next_idx <= command_count) + unsigned i; + + for (i = 1; i < countof(commands); ++i) { - cmp = strcasecmp (commands[prev_idx].name, commands[next_idx].name); - if (cmp > 0) + if (strcasecmp (commands[i - 1].name, commands[i].name) > 0) { mu_assert ("FAILED", false); break; - } - else - { - prev_idx ++; - next_idx ++; } } return NULL; } const char * -test_cmd_spec_restrict_file_names() +test_cmd_spec_restrict_file_names(void) { - int i; - struct { - char *val; + unsigned i; + static const struct { + const char *val; int expected_restrict_files_os; int expected_restrict_files_ctrl; int expected_restrict_files_case; @@ -1621,11 +1818,11 @@ test_cmd_spec_restrict_file_names() { "windows,lowercase", restrict_windows, true, restrict_lowercase, true }, { "unix,nocontrol,lowercase,", restrict_unix, false, restrict_lowercase, true }, }; - - for (i = 0; i < sizeof(test_array)/sizeof(test_array[0]); ++i) + + for (i = 0; i < countof(test_array); ++i) { bool res; - + defaults(); res = cmd_spec_restrict_file_names ("dummy", test_array[i].val, NULL); @@ -1635,10 +1832,10 @@ test_cmd_spec_restrict_file_names() fprintf (stderr, "opt.restrict_files_ctrl: %d\n", opt.restrict_files_ctrl); fflush (stderr); fprintf (stderr, "opt.restrict_files_case: %d\n", opt.restrict_files_case); fflush (stderr); */ - mu_assert ("test_cmd_spec_restrict_file_names: wrong result", + mu_assert ("test_cmd_spec_restrict_file_names: wrong result", res == test_array[i].result - && opt.restrict_files_os == test_array[i].expected_restrict_files_os - && opt.restrict_files_ctrl == test_array[i].expected_restrict_files_ctrl + && opt.restrict_files_os == test_array[i].expected_restrict_files_os + && opt.restrict_files_ctrl == test_array[i].expected_restrict_files_ctrl && opt.restrict_files_case == test_array[i].expected_restrict_files_case); }