#include "test.h"
#endif
+#ifdef __VMS
+# include "vms.h"
+#endif /* def __VMS */
+
extern char *version_string;
/* Forward decls. */
while (p < end && c_isdigit (*p))
++p;
if (p < end && *p == '.')
- ++p;
+ ++p;
while (p < end && c_isdigit (*p))
++p;
}
{
char *copy;
BOUNDED_TO_ALLOCA(b, e, copy);
- logprintf (LOG_ALWAYS, "%s%s\n", prefix,
+ logprintf (LOG_ALWAYS, "%s%s\n", prefix,
quotearg_style (escape_quoting_style, copy));
}
bool add_slash = (opt.dir_prefix[prefix_length - 1] != '/');
int total_length;
- if (add_slash)
+ if (add_slash)
++prefix_length;
- total_length = prefix_length + (value.e - value.b);
+ total_length = prefix_length + (value.e - value.b);
*filename = xmalloc (total_length + 1);
strcpy (*filename, opt.dir_prefix);
- if (add_slash)
+ if (add_slash)
(*filename)[prefix_length - 1] = '/';
memcpy (*filename + prefix_length, value.b, (value.e - value.b));
(*filename)[total_length] = '\0';
existence after having begun to download
(needed in gethttp for when connection is
interrupted/restarted. */
- bool timestamp_checked; /* true if pre-download time-stamping checks
+ bool timestamp_checked; /* true if pre-download time-stamping checks
* have already been performed */
char *orig_file_name; /* name of file to compare for time-stamping
* (might be != local_file if -K is set) */
wgint orig_file_size; /* size of file to compare for time-stamping */
- time_t orig_file_tstamp; /* time-stamp of file to compare for
+ time_t orig_file_tstamp; /* time-stamp of file to compare for
* time-stamping */
};
&& (c_isspace (line[sizeof (string_constant) - 1]) \
|| !line[sizeof (string_constant) - 1]))
+#ifdef __VMS
#define SET_USER_AGENT(req) do { \
if (!opt.useragent) \
request_set_header (req, "User-Agent", \
- aprintf ("Wget/%s", version_string), rel_value); \
+ aprintf ("Wget/%s (VMS %s %s)", \
+ version_string, vms_arch(), vms_vers()), \
+ rel_value); \
else if (*opt.useragent) \
request_set_header (req, "User-Agent", opt.useragent, rel_none); \
} while (0)
+#else /* def __VMS */
+#define SET_USER_AGENT(req) do { \
+ if (!opt.useragent) \
+ request_set_header (req, "User-Agent", \
+ aprintf ("Wget/%s (%s)", \
+ version_string, OS_TYPE), \
+ rel_value); \
+ else if (*opt.useragent) \
+ request_set_header (req, "User-Agent", opt.useragent, rel_none); \
+} while (0)
+#endif /* def __VMS [else] */
/* The flags that allow clobbering the file (opening with "wb").
Defined here to avoid repetition later. #### This will require
If PROXY is non-NULL, the connection will be made to the proxy
server, and u->url will be requested. */
static uerr_t
-gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy)
+gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
+ struct iri *iri)
{
struct request *req;
sock = pconn.socket;
using_ssl = pconn.ssl;
logprintf (LOG_VERBOSE, _("Reusing existing connection to %s:%d.\n"),
- quotearg_style (escape_quoting_style, pconn.host),
+ quotearg_style (escape_quoting_style, pconn.host),
pconn.port);
DEBUGP (("Reusing fd %d.\n", sock));
if (pconn.authorized)
if (conn->scheme == SCHEME_HTTPS)
{
- if (!ssl_connect (sock) || !ssl_check_certificate (sock, u->host))
+ if (!ssl_connect_wget (sock))
{
fd_close (sock);
return CONSSLERR;
}
+ else if (!ssl_check_certificate (sock, u->host))
+ {
+ fd_close (sock);
+ return VERIFCERTERR;
+ }
using_ssl = true;
}
#endif /* HAVE_SSL */
print_server_response (resp, " ");
}
+ if (!opt.ignore_length
+ && resp_header_copy (resp, "Content-Length", hdrval, sizeof (hdrval)))
+ {
+ wgint parsed;
+ errno = 0;
+ parsed = str_to_wgint (hdrval, NULL, 10);
+ if (parsed == WGINT_MAX && errno == ERANGE)
+ {
+ /* Out of range.
+ #### If Content-Length is out of range, it most likely
+ means that the file is larger than 2G and that we're
+ compiled without LFS. In that case we should probably
+ refuse to even attempt to download the file. */
+ contlen = -1;
+ }
+ else if (parsed < 0)
+ {
+ /* Negative Content-Length; nonsensical, so we can't
+ assume any information about the content to receive. */
+ contlen = -1;
+ }
+ else
+ contlen = parsed;
+ }
+
/* Check for keep-alive related responses. */
if (!inhibit_keep_alive && contlen != -1)
{
}
}
+ /* Handle (possibly multiple instances of) the Set-Cookie header. */
+ if (opt.cookies)
+ {
+ int scpos;
+ const char *scbeg, *scend;
+ /* The jar should have been created by now. */
+ assert (wget_cookie_jar != NULL);
+ for (scpos = 0;
+ (scpos = resp_header_locate (resp, "Set-Cookie", scpos,
+ &scbeg, &scend)) != -1;
+ ++scpos)
+ {
+ char *set_cookie; BOUNDED_TO_ALLOCA (scbeg, scend, set_cookie);
+ cookie_handle_set_cookie (wget_cookie_jar, u->host, u->port,
+ u->path, set_cookie);
+ }
+ }
+
if (keep_alive)
/* The server has promised that it will not close the connection
when we're done. This means that we can register it. */
pconn.authorized = true;
}
- /* Determine the local filename if needed. Notice that if -O is used
+ /* Determine the local filename if needed. Notice that if -O is used
* hstat.local_file is set by http_loop to the argument of -O. */
if (!hs->local_file)
{
/* Honor Content-Disposition whether possible. */
if (!opt.content_disposition
- || !resp_header_copy (resp, "Content-Disposition",
+ || !resp_header_copy (resp, "Content-Disposition",
hdrval, sizeof (hdrval))
|| !parse_content_disposition (hdrval, &hs->local_file))
{
- /* The Content-Disposition header is missing or broken.
+ /* The Content-Disposition header is missing or broken.
* Choose unique file name according to given URL. */
hs->local_file = url_file_name (u);
}
}
-
+
/* TODO: perform this check only once. */
if (!hs->existence_checked && file_exists_p (hs->local_file))
{
if (opt.timestamping && !hs->timestamp_checked)
{
size_t filename_len = strlen (hs->local_file);
- char *filename_plus_orig_suffix = alloca (filename_len + sizeof (".orig"));
+ char *filename_plus_orig_suffix = alloca (filename_len + sizeof (ORIG_SFX));
bool local_dot_orig_file_exists = false;
char *local_filename = NULL;
struct_stat st;
--hniksic */
memcpy (filename_plus_orig_suffix, hs->local_file, filename_len);
memcpy (filename_plus_orig_suffix + filename_len,
- ".orig", sizeof (".orig"));
+ ORIG_SFX, sizeof (ORIG_SFX));
/* Try to stat() the .orig file. */
if (stat (filename_plus_orig_suffix, &st) == 0)
local_dot_orig_file_exists = true;
local_filename = filename_plus_orig_suffix;
}
- }
+ }
if (!local_dot_orig_file_exists)
/* Couldn't stat() <file>.orig, so try to stat() <file>. */
}
}
- if (!opt.ignore_length
- && resp_header_copy (resp, "Content-Length", hdrval, sizeof (hdrval)))
- {
- wgint parsed;
- errno = 0;
- parsed = str_to_wgint (hdrval, NULL, 10);
- if (parsed == WGINT_MAX && errno == ERANGE)
- {
- /* Out of range.
- #### If Content-Length is out of range, it most likely
- means that the file is larger than 2G and that we're
- compiled without LFS. In that case we should probably
- refuse to even attempt to download the file. */
- contlen = -1;
- }
- else if (parsed < 0)
- {
- /* Negative Content-Length; nonsensical, so we can't
- assume any information about the content to receive. */
- contlen = -1;
- }
- else
- contlen = parsed;
- }
-
request_free (req);
hs->statcode = statcode;
char *tmp = strchr (type, ';');
if (tmp)
{
+ /* sXXXav: only needed if IRI support is enabled */
+ char *tmp2 = tmp + 1;
+
while (tmp > type && c_isspace (tmp[-1]))
--tmp;
*tmp = '\0';
+
+ /* Try to get remote encoding if needed */
+ if (opt.enable_iri && !opt.encoding_remote)
+ {
+ tmp = parse_charset (tmp2);
+ if (tmp)
+ set_content_encoding (iri, tmp);
+ }
}
}
hs->newloc = resp_header_strdup (resp, "Location");
hs->remote_time = resp_header_strdup (resp, "Last-Modified");
- /* Handle (possibly multiple instances of) the Set-Cookie header. */
- if (opt.cookies)
- {
- int scpos;
- const char *scbeg, *scend;
- /* The jar should have been created by now. */
- assert (wget_cookie_jar != NULL);
- for (scpos = 0;
- (scpos = resp_header_locate (resp, "Set-Cookie", scpos,
- &scbeg, &scend)) != -1;
- ++scpos)
- {
- char *set_cookie; BOUNDED_TO_ALLOCA (scbeg, scend, set_cookie);
- cookie_handle_set_cookie (wget_cookie_jar, u->host, u->port,
- u->path, set_cookie);
- }
- }
-
if (resp_header_copy (resp, "Content-Range", hdrval, sizeof (hdrval)))
{
wgint first_byte_pos, last_byte_pos, entity_length;
content-type. */
if (!type ||
0 == strncasecmp (type, TEXTHTML_S, strlen (TEXTHTML_S)) ||
- 0 == strncasecmp (type, TEXTXHTML_S, strlen (TEXTXHTML_S)))
+ 0 == strncasecmp (type, TEXTXHTML_S, strlen (TEXTXHTML_S)))
*dt |= TEXTHTML;
else
*dt &= ~TEXTHTML;
else
*dt &= ~TEXTCSS;
- if (opt.html_extension)
+ if (opt.adjust_extension)
{
if (*dt & TEXTHTML)
- /* -E / --html-extension / html_extension = on was specified,
+ /* -E / --adjust-extension / adjust_extension = on was specified,
and this is a text/html file. If some case-insensitive
variation on ".htm[l]" isn't already the file's suffix,
tack on ".html". */
return RETRFINISHED;
}
+/* 2005-06-17 SMS.
+ For VMS, define common fopen() optional arguments.
+*/
+#ifdef __VMS
+# define FOPEN_OPT_ARGS "fop=sqo", "acc", acc_cb, &open_id
+# define FOPEN_BIN_FLAG 3
+#else /* def __VMS */
+# define FOPEN_BIN_FLAG true
+#endif /* def __VMS [else] */
+
/* Open the local file. */
if (!output_stream)
{
if (opt.backups)
rotate_backups (hs->local_file);
if (hs->restval)
- fp = fopen (hs->local_file, "ab");
+ {
+#ifdef __VMS
+ int open_id;
+
+ open_id = 21;
+ fp = fopen (hs->local_file, "ab", FOPEN_OPT_ARGS);
+#else /* def __VMS */
+ fp = fopen (hs->local_file, "ab");
+#endif /* def __VMS [else] */
+ }
else if (ALLOW_CLOBBER)
- fp = fopen (hs->local_file, "wb");
+ {
+#ifdef __VMS
+ int open_id;
+
+ open_id = 22;
+ fp = fopen (hs->local_file, "wb", FOPEN_OPT_ARGS);
+#else /* def __VMS */
+ fp = fopen (hs->local_file, "wb");
+#endif /* def __VMS [else] */
+ }
else
{
- fp = fopen_excl (hs->local_file, true);
+ fp = fopen_excl (hs->local_file, FOPEN_BIN_FLAG);
if (!fp && errno == EEXIST)
{
/* We cannot just invent a new name and use it (which is
/* Print fetch message, if opt.verbose. */
if (opt.verbose)
{
- logprintf (LOG_NOTQUIET, _("Saving to: %s\n"),
+ logprintf (LOG_NOTQUIET, _("Saving to: %s\n"),
HYPHENP (hs->local_file) ? quote ("STDOUT") : quote (hs->local_file));
}
-
+
/* This confuses the timestamping code that checks for file size.
#### The timestamping code should be smarter about file size. */
if (opt.save_headers && hs->restval == 0)
retried, and retried, and retried, and... */
uerr_t
http_loop (struct url *u, char **newloc, char **local_file, const char *referer,
- int *dt, struct url *proxy)
+ int *dt, struct url *proxy, struct iri *iri)
{
int count;
bool got_head = false; /* used for time-stamping and filename detection */
uerr_t err, ret = TRYLIMEXC;
time_t tmr = -1; /* remote time-stamp */
struct http_stat hstat; /* HTTP status */
- struct_stat st;
+ struct_stat st;
bool send_head_first = true;
char *file_name;
/* Assert that no value for *LOCAL_FILE was passed. */
assert (local_file == NULL || *local_file == NULL);
-
+
/* Set LOCAL_FILE parameter. */
if (local_file && opt.output_document)
*local_file = HYPHENP (opt.output_document) ? NULL : xstrdup (opt.output_document);
-
+
/* Reset NEWLOC parameter. */
*newloc = NULL;
retrieve the file. But if the output_document was given, then this
test was already done and the file didn't exist. Hence the !opt.output_document */
logprintf (LOG_VERBOSE, _("\
-File %s already there; not retrieving.\n\n"),
+File %s already there; not retrieving.\n\n"),
quote (hstat.local_file));
/* If the file is there, we suppose it's retrieved OK. */
*dt |= RETROKF;
/* Reset the counter. */
count = 0;
-
+
/* Reset the document type. */
*dt = 0;
-
+
/* Skip preliminary HEAD request if we're not in spider mode AND
* if -O was given or HTTP Content-Disposition support is disabled. */
if (!opt.spider
&& (got_name || !opt.content_disposition))
send_head_first = false;
- /* Send preliminary HEAD request if -N is given and we have an existing
+ /* Send preliminary HEAD request if -N is given and we have an existing
* destination file. */
file_name = url_file_name (u);
- if (opt.timestamping
+ if (opt.timestamping
&& !opt.content_disposition
&& file_exists_p (file_name))
send_head_first = true;
xfree (file_name);
-
+
/* THE loop */
do
{
/* Increment the pass counter. */
++count;
sleep_between_retrievals (count);
-
+
/* Get the current time string. */
tms = datetime_str (time (NULL));
-
+
if (opt.spider && !got_head)
logprintf (LOG_VERBOSE, _("\
Spider mode enabled. Check if remote file exists.\n"));
if (opt.verbose)
{
char *hurl = url_string (u, URL_AUTH_HIDE_PASSWD);
-
- if (count > 1)
+
+ if (count > 1)
{
char tmp[256];
sprintf (tmp, _("(try:%2d)"), count);
logprintf (LOG_NOTQUIET, "--%s-- %s %s\n",
tms, tmp, hurl);
}
- else
+ else
{
logprintf (LOG_NOTQUIET, "--%s-- %s\n",
tms, hurl);
}
-
+
#ifdef WINDOWS
ws_changetitle (hurl);
#endif
/* Default document type is empty. However, if spider mode is
on or time-stamping is employed, HEAD_ONLY commands is
encoded within *dt. */
- if (send_head_first && !got_head)
+ if (send_head_first && !got_head)
*dt |= HEAD_ONLY;
else
*dt &= ~HEAD_ONLY;
*dt &= ~SEND_NOCACHE;
/* Try fetching the document, or at least its head. */
- err = gethttp (u, &hstat, dt, proxy);
+ err = gethttp (u, &hstat, dt, proxy, iri);
/* Time? */
tms = datetime_str (time (NULL));
-
+
/* Get the new location (with or without the redirection). */
if (hstat.newloc)
*newloc = xstrdup (hstat.newloc);
logputs (LOG_VERBOSE, "\n");
logprintf (LOG_NOTQUIET, _("Cannot write to %s (%s).\n"),
quote (hstat.local_file), strerror (errno));
- case HOSTERR: case CONIMPOSSIBLE: case PROXERR: case AUTHFAILED:
- case SSLINITFAILED: case CONTNOTSUPPORTED:
+ case HOSTERR: case CONIMPOSSIBLE: case PROXERR: case AUTHFAILED:
+ case SSLINITFAILED: case CONTNOTSUPPORTED: case VERIFCERTERR:
/* Fatal errors just return from the function. */
ret = err;
goto exit;
hstat.statcode);
ret = WRONGCODE;
}
- else
+ else
{
ret = NEWLOCATION;
}
/* All possibilities should have been exhausted. */
abort ();
}
-
+
if (!(*dt & RETROKF))
{
char *hurl = NULL;
continue;
}
/* Maybe we should always keep track of broken links, not just in
- * spider mode. */
- else if (opt.spider)
+ * spider mode.
+ * Don't log error if it was UTF-8 encoded because we will try
+ * once unencoded. */
+ else if (opt.spider && !iri->utf8_encode)
{
/* #### Again: ugly ugly ugly! */
- if (!hurl)
+ if (!hurl)
hurl = url_string (u, URL_AUTH_HIDE_PASSWD);
nonexisting_url (hurl);
logprintf (LOG_NOTQUIET, _("\
else
{
logprintf (LOG_NOTQUIET, _("%s ERROR %d: %s.\n"),
- tms, hstat.statcode,
+ tms, hstat.statcode,
quotearg_style (escape_quoting_style, hstat.error));
}
logputs (LOG_VERBOSE, "\n");
if (*dt & HEAD_ONLY)
time_came_from_head = true;
}
-
+
if (send_head_first)
{
/* The time-stamping section. */
we're supposed to
download already exists. */
{
- if (hstat.remote_time &&
+ if (hstat.remote_time &&
tmr != (time_t) (-1))
{
/* Now time-stamping can be used validly.
download procedure is resumed. */
if (hstat.orig_file_tstamp >= tmr)
{
- if (hstat.contlen == -1
+ if (hstat.contlen == -1
|| hstat.orig_file_size == hstat.contlen)
{
logprintf (LOG_VERBOSE, _("\
logputs (LOG_VERBOSE, "\n");
}
}
-
+
/* free_hstat (&hstat); */
hstat.timestamp_checked = true;
}
-
+
if (opt.spider)
{
bool finished = true;
Remote file exists and could contain links to other resources -- retrieving.\n\n"));
finished = false;
}
- else
+ else
{
logprintf (LOG_VERBOSE, _("\
Remote file exists but does not contain any link -- not retrieving.\n\n"));
Remote file exists and could contain further links,\n\
but recursion is disabled -- not retrieving.\n\n"));
}
- else
+ else
{
logprintf (LOG_VERBOSE, _("\
Remote file exists.\n\n"));
}
ret = RETROK; /* RETRUNNEEDED is not for caller. */
}
-
+
if (finished)
{
- logprintf (LOG_NONVERBOSE,
- _("%s URL:%s %2d %s\n"),
+ logprintf (LOG_NONVERBOSE,
+ _("%s URL: %s %2d %s\n"),
tms, u->url, hstat.statcode,
hstat.message ? quotearg_style (escape_quoting_style, hstat.message) : "");
goto exit;
continue;
} /* send_head_first */
} /* !got_head */
-
+
if ((tmr != (time_t) (-1))
&& ((hstat.len == hstat.contlen) ||
((hstat.res == 0) && (hstat.contlen == -1))))
&& hstat.remote_time && hstat.remote_time[0])
{
newtmr = http_atotm (hstat.remote_time);
- if (newtmr != -1)
+ if (newtmr != (time_t)-1)
tmr = newtmr;
}
touch (fl, tmr);
bool write_to_stdout = (opt.output_document && HYPHENP (opt.output_document));
logprintf (LOG_VERBOSE,
- write_to_stdout
+ write_to_stdout
? _("%s (%s) - written to stdout %s[%s/%s]\n\n")
: _("%s (%s) - %s saved [%s/%s]\n\n"),
tms, tmrate,
else if (hstat.res == 0) /* No read error */
{
if (hstat.contlen == -1) /* We don't know how much we were supposed
- to get, so assume we succeeded. */
+ to get, so assume we succeeded. */
{
if (*dt & RETROKF)
{
write_to_stdout
? _("%s (%s) - written to stdout %s[%s]\n\n")
: _("%s (%s) - %s saved [%s]\n\n"),
- tms, tmrate,
+ tms, tmrate,
write_to_stdout ? "" : quote (hstat.local_file),
number_to_static_string (hstat.len));
logprintf (LOG_NONVERBOSE,
downloaded_file(FILE_DOWNLOADED_AND_HTML_EXTENSION_ADDED, hstat.local_file);
else
downloaded_file(FILE_DOWNLOADED_NORMALLY, hstat.local_file);
-
+
ret = RETROK;
goto exit;
}
while (!opt.ntry || (count < opt.ntry));
exit:
- if (ret == RETROK)
+ if (ret == RETROK)
*local_file = xstrdup (hstat.local_file);
free_hstat (&hstat);
-
+
return ret;
}
\f
{
int i;
struct {
- char *hdrval;
+ char *hdrval;
char *opt_dir_prefix;
char *filename;
bool result;
{ "attachment", NULL, NULL, false },
{ "attachment", "somedir", NULL, false },
};
-
- for (i = 0; i < sizeof(test_array)/sizeof(test_array[0]); ++i)
+
+ for (i = 0; i < sizeof(test_array)/sizeof(test_array[0]); ++i)
{
char *filename;
bool res;
opt.dir_prefix = test_array[i].opt_dir_prefix;
res = parse_content_disposition (test_array[i].hdrval, &filename);
- mu_assert ("test_parse_content_disposition: wrong result",
+ mu_assert ("test_parse_content_disposition: wrong result",
res == test_array[i].result
- && (res == false
+ && (res == false
|| 0 == strcmp (test_array[i].filename, filename)));
}