#include "utils.h"
#include "hash.h"
#include "ptimer.h"
+#include "res.h"
static struct hash_table *dl_file_url_map;
struct hash_table *dl_url_file_map;
char *file = file_array[i];
/* Determine the URL of the HTML file. get_urls_html will need
- it. */
+ it. */
url = hash_table_get (dl_file_url_map, file);
if (!url)
- {
- DEBUGP (("Apparently %s has been removed.\n", file));
- continue;
- }
+ {
+ DEBUGP (("Apparently %s has been removed.\n", file));
+ continue;
+ }
DEBUGP (("Scanning %s (from %s)\n", file, url));
links that have been followed from other files. */
for (cur_url = urls; cur_url; cur_url = cur_url->next)
- {
- char *local_name;
- struct url *u = cur_url->url;
-
- if (cur_url->link_base_p)
- {
- /* Base references have been resolved by our parser, so
- we turn the base URL into an empty string. (Perhaps
- we should remove the tag entirely?) */
- cur_url->convert = CO_NULLIFY_BASE;
- continue;
- }
-
- /* We decide the direction of conversion according to whether
- a URL was downloaded. Downloaded URLs will be converted
- ABS2REL, whereas non-downloaded will be converted REL2ABS. */
- local_name = hash_table_get (dl_url_file_map, u->url);
-
- /* Decide on the conversion type. */
- if (local_name)
- {
- /* We've downloaded this URL. Convert it to relative
+ {
+ char *local_name;
+ struct url *u = cur_url->url;
+
+ if (cur_url->link_base_p)
+ {
+ /* Base references have been resolved by our parser, so
+ we turn the base URL into an empty string. (Perhaps
+ we should remove the tag entirely?) */
+ cur_url->convert = CO_NULLIFY_BASE;
+ continue;
+ }
+
+ /* We decide the direction of conversion according to whether
+ a URL was downloaded. Downloaded URLs will be converted
+ ABS2REL, whereas non-downloaded will be converted REL2ABS. */
+ local_name = hash_table_get (dl_url_file_map, u->url);
+
+ /* Decide on the conversion type. */
+ if (local_name)
+ {
+ /* We've downloaded this URL. Convert it to relative
form. We do this even if the URL already is in
relative form, because our directory structure may
not be identical to that on the server (think `-nd',
`--cut-dirs', etc.) */
- cur_url->convert = CO_CONVERT_TO_RELATIVE;
- cur_url->local_name = xstrdup (local_name);
- DEBUGP (("will convert url %s to local %s\n", u->url, local_name));
- }
- else
- {
- /* We haven't downloaded this URL. If it's not already
+ cur_url->convert = CO_CONVERT_TO_RELATIVE;
+ cur_url->local_name = xstrdup (local_name);
+ DEBUGP (("will convert url %s to local %s\n", u->url, local_name));
+ }
+ else
+ {
+ /* We haven't downloaded this URL. If it's not already
complete (including a full host name), convert it to
that form, so it can be reached while browsing this
HTML locally. */
- if (!cur_url->link_complete_p)
- cur_url->convert = CO_CONVERT_TO_COMPLETE;
- cur_url->local_name = NULL;
- DEBUGP (("will convert url %s to complete\n", u->url));
- }
- }
+ if (!cur_url->link_complete_p)
+ cur_url->convert = CO_CONVERT_TO_COMPLETE;
+ cur_url->local_name = NULL;
+ DEBUGP (("will convert url %s to complete\n", u->url));
+ }
+ }
/* Convert the links in the file. */
convert_links (file, urls);
secs = ptimer_measure (timer);
ptimer_destroy (timer);
logprintf (LOG_VERBOSE, _("Converted %d files in %s seconds.\n"),
- file_count, print_decimal (secs));
+ file_count, print_decimal (secs));
}
static void write_backup_file (const char *, downloaded_file_t);
static const char *replace_attr (const char *, int, FILE *, const char *);
static const char *replace_attr_refresh_hack (const char *, int, FILE *,
- const char *, int);
+ const char *, int);
static char *local_quote_string (const char *);
static char *construct_relative (const char *, const char *);
struct urlpos *dry;
for (dry = links; dry; dry = dry->next)
if (dry->convert != CO_NOCONVERT)
- ++dry_count;
+ ++dry_count;
if (!dry_count)
{
- logputs (LOG_VERBOSE, _("nothing to do.\n"));
- return;
+ logputs (LOG_VERBOSE, _("nothing to do.\n"));
+ return;
}
}
if (!fm)
{
logprintf (LOG_NOTQUIET, _("Cannot convert links in %s: %s\n"),
- file, strerror (errno));
+ file, strerror (errno));
return;
}
if (unlink (file) < 0 && errno != ENOENT)
{
logprintf (LOG_NOTQUIET, _("Unable to delete `%s': %s\n"),
- file, strerror (errno));
+ file, strerror (errno));
read_file_free (fm);
return;
}
if (!fp)
{
logprintf (LOG_NOTQUIET, _("Cannot convert links in %s: %s\n"),
- file, strerror (errno));
+ file, strerror (errno));
read_file_free (fm);
return;
}
char *url_start = fm->content + link->pos;
if (link->pos >= fm->length)
- {
- DEBUGP (("Something strange is going on. Please investigate."));
- break;
- }
+ {
+ DEBUGP (("Something strange is going on. Please investigate."));
+ break;
+ }
/* If the URL is not to be converted, skip it. */
if (link->convert == CO_NOCONVERT)
- {
- DEBUGP (("Skipping %s at position %d.\n", link->url->url, link->pos));
- continue;
- }
+ {
+ DEBUGP (("Skipping %s at position %d.\n", link->url->url, link->pos));
+ continue;
+ }
/* Echo the file contents, up to the offending URL's opening
quote, to the outfile. */
p = url_start;
switch (link->convert)
- {
- case CO_CONVERT_TO_RELATIVE:
- /* Convert absolute URL to relative. */
- {
- char *newname = construct_relative (file, link->local_name);
- char *quoted_newname = local_quote_string (newname);
-
- if (!link->link_refresh_p)
- p = replace_attr (p, link->size, fp, quoted_newname);
- else
- p = replace_attr_refresh_hack (p, link->size, fp, quoted_newname,
- link->refresh_timeout);
-
- DEBUGP (("TO_RELATIVE: %s to %s at position %d in %s.\n",
- link->url->url, newname, link->pos, file));
- xfree (newname);
- xfree (quoted_newname);
- ++to_file_count;
- break;
- }
- case CO_CONVERT_TO_COMPLETE:
- /* Convert the link to absolute URL. */
- {
- char *newlink = link->url->url;
- char *quoted_newlink = html_quote_string (newlink);
-
- if (!link->link_refresh_p)
- p = replace_attr (p, link->size, fp, quoted_newlink);
- else
- p = replace_attr_refresh_hack (p, link->size, fp, quoted_newlink,
- link->refresh_timeout);
-
- DEBUGP (("TO_COMPLETE: <something> to %s at position %d in %s.\n",
- newlink, link->pos, file));
- xfree (quoted_newlink);
- ++to_url_count;
- break;
- }
- case CO_NULLIFY_BASE:
- /* Change the base href to "". */
- p = replace_attr (p, link->size, fp, "");
- break;
- case CO_NOCONVERT:
- abort ();
- break;
- }
+ {
+ case CO_CONVERT_TO_RELATIVE:
+ /* Convert absolute URL to relative. */
+ {
+ char *newname = construct_relative (file, link->local_name);
+ char *quoted_newname = local_quote_string (newname);
+
+ if (!link->link_refresh_p)
+ p = replace_attr (p, link->size, fp, quoted_newname);
+ else
+ p = replace_attr_refresh_hack (p, link->size, fp, quoted_newname,
+ link->refresh_timeout);
+
+ DEBUGP (("TO_RELATIVE: %s to %s at position %d in %s.\n",
+ link->url->url, newname, link->pos, file));
+ xfree (newname);
+ xfree (quoted_newname);
+ ++to_file_count;
+ break;
+ }
+ case CO_CONVERT_TO_COMPLETE:
+ /* Convert the link to absolute URL. */
+ {
+ char *newlink = link->url->url;
+ char *quoted_newlink = html_quote_string (newlink);
+
+ if (!link->link_refresh_p)
+ p = replace_attr (p, link->size, fp, quoted_newlink);
+ else
+ p = replace_attr_refresh_hack (p, link->size, fp, quoted_newlink,
+ link->refresh_timeout);
+
+ DEBUGP (("TO_COMPLETE: <something> to %s at position %d in %s.\n",
+ newlink, link->pos, file));
+ xfree (quoted_newlink);
+ ++to_url_count;
+ break;
+ }
+ case CO_NULLIFY_BASE:
+ /* Change the base href to "". */
+ p = replace_attr (p, link->size, fp, "");
+ break;
+ case CO_NOCONVERT:
+ abort ();
+ break;
+ }
}
/* Output the rest of the file. */
for (b = basefile, l = linkfile; *b == *l && *b != '\0'; ++b, ++l)
{
if (*b == '/')
- start = (b - basefile) + 1;
+ start = (b - basefile) + 1;
}
basefile += start;
linkfile += start;
for (b = basefile; *b; b++)
{
if (*b == '/')
- ++basedirs;
+ ++basedirs;
}
/* Construct LINK as explained above. */
if (downloaded_file_return == FILE_DOWNLOADED_AND_HTML_EXTENSION_ADDED)
{
/* Just write "orig" over "html". We need to do it this way
- because when we're checking to see if we've downloaded the
- file before (to see if we can skip downloading it), we don't
- know if it's a text/html file. Therefore we don't know yet
- at that stage that -E is going to cause us to tack on
- ".html", so we need to compare vs. the original URL plus
- ".orig", not the original URL plus ".html.orig". */
+ because when we're checking to see if we've downloaded the
+ file before (to see if we can skip downloading it), we don't
+ know if it's a text/html file. Therefore we don't know yet
+ at that stage that -E is going to cause us to tack on
+ ".html", so we need to compare vs. the original URL plus
+ ".orig", not the original URL plus ".html.orig". */
filename_plus_orig_suffix = alloca (filename_len + 1);
strcpy (filename_plus_orig_suffix, file);
strcpy ((filename_plus_orig_suffix + filename_len) - 4, "orig");
{
/* Rename <file> to <file>.orig before former gets written over. */
if (rename (file, filename_plus_orig_suffix) != 0)
- logprintf (LOG_NOTQUIET, _("Cannot back up %s as %s: %s\n"),
- file, filename_plus_orig_suffix, strerror (errno));
+ logprintf (LOG_NOTQUIET, _("Cannot back up %s as %s: %s\n"),
+ file, filename_plus_orig_suffix, strerror (errno));
/* Remember that we've already written a .orig backup for this file.
- Note that we never free this memory since we need it till the
- convert_all_links() call, which is one of the last things the
- program does before terminating. BTW, I'm not sure if it would be
- safe to just set 'converted_file_ptr->string' to 'file' below,
- rather than making a copy of the string... Another note is that I
- thought I could just add a field to the urlpos structure saying
- that we'd written a .orig file for this URL, but that didn't work,
- so I had to make this separate list.
- -- Dan Harkless <wget@harkless.org>
+ Note that we never free this memory since we need it till the
+ convert_all_links() call, which is one of the last things the
+ program does before terminating. BTW, I'm not sure if it would be
+ safe to just set 'converted_file_ptr->string' to 'file' below,
+ rather than making a copy of the string... Another note is that I
+ thought I could just add a field to the urlpos structure saying
+ that we'd written a .orig file for this URL, but that didn't work,
+ so I had to make this separate list.
+ -- Dan Harkless <wget@harkless.org>
This [adding a field to the urlpos structure] didn't work
because convert_file() is called from convert_all_links at
the end of the retrieval with a freshly built new urlpos
list.
- -- Hrvoje Niksic <hniksic@xemacs.org>
+ -- Hrvoje Niksic <hniksic@xemacs.org>
*/
string_set_add (converted_files, file);
}
replace_attr (const char *p, int size, FILE *fp, const char *new_text)
{
bool quote_flag = false;
- char quote_char = '\"'; /* use "..." for quoting, unless the
- original value is quoted, in which
- case reuse its quoting char. */
+ char quote_char = '\"'; /* use "..." for quoting, unless the
+ original value is quoted, in which
+ case reuse its quoting char. */
const char *frag_beg, *frag_end;
/* Structure of our string is:
quote_char = *p;
quote_flag = true;
++p;
- size -= 2; /* disregard opening and closing quote */
+ size -= 2; /* disregard opening and closing quote */
}
putc (quote_char, fp);
fputs (new_text, fp);
static const char *
replace_attr_refresh_hack (const char *p, int size, FILE *fp,
- const char *new_text, int timeout)
+ const char *new_text, int timeout)
{
/* "0; URL=..." */
char *new_with_timeout = (char *)alloca (numdigit (timeout)
- + 6 /* "; URL=" */
- + strlen (new_text)
- + 1);
+ + 6 /* "; URL=" */
+ + strlen (new_text)
+ + 1);
sprintf (new_with_timeout, "%d; URL=%s", timeout, new_text);
return replace_attr (p, size, fp, new_with_timeout);
for (; beg < end; beg++)
{
switch (*beg)
- {
- case '&':
- saw_amp = true;
- break;
- case '#':
- if (!saw_amp)
- {
- *bp = beg;
- *ep = end;
- return true;
- }
- /* fallthrough */
- default:
- saw_amp = false;
- }
+ {
+ case '&':
+ saw_amp = true;
+ break;
+ case '#':
+ if (!saw_amp)
+ {
+ *bp = beg;
+ *ep = end;
+ return true;
+ }
+ /* fallthrough */
+ default:
+ saw_amp = false;
+ }
}
return false;
}
switch (*from)
{
case '%':
- *to++ = '%';
- *to++ = '2';
- *to++ = '5';
- break;
+ *to++ = '%';
+ *to++ = '2';
+ *to++ = '5';
+ break;
case '#':
- *to++ = '%';
- *to++ = '2';
- *to++ = '3';
- break;
+ *to++ = '%';
+ *to++ = '2';
+ *to++ = '3';
+ break;
case '?':
- if (opt.html_extension)
- {
- *to++ = '%';
- *to++ = '3';
- *to++ = 'F';
- break;
- }
- /* fallthrough */
+ if (opt.html_extension)
+ {
+ *to++ = '%';
+ *to++ = '3';
+ *to++ = 'F';
+ break;
+ }
+ /* fallthrough */
default:
- *to++ = *from;
+ *to++ = *from;
}
*to = '\0';
downloaded_html_list, and downloaded_html_set. Other code calls
these functions to let us know that a file has been downloaded. */
-#define ENSURE_TABLES_EXIST do { \
- if (!dl_file_url_map) \
- dl_file_url_map = make_string_hash_table (0); \
- if (!dl_url_file_map) \
- dl_url_file_map = make_string_hash_table (0); \
+#define ENSURE_TABLES_EXIST do { \
+ if (!dl_file_url_map) \
+ dl_file_url_map = make_string_hash_table (0); \
+ if (!dl_url_file_map) \
+ dl_url_file_map = make_string_hash_table (0); \
} while (0)
/* Return true if S1 and S2 are the same, except for "/index.html".
{
/* Can't use hash_table_iter_* because the table mutates while mapping. */
hash_table_for_each (dl_url_file_map, dissociate_urls_from_file_mapper,
- (char *) file);
+ (char *) file);
}
/* Register that URL has been successfully downloaded to FILE. This
if (hash_table_get_pair (dl_file_url_map, file, &old_file, &old_url))
{
if (0 == strcmp (url, old_url))
- /* We have somehow managed to download the same URL twice.
- Nothing to do. */
- return;
+ /* We have somehow managed to download the same URL twice.
+ Nothing to do. */
+ return;
if (match_except_index (url, old_url)
- && !hash_table_contains (dl_url_file_map, url))
- /* The two URLs differ only in the "index.html" ending. For
- example, one is "http://www.server.com/", and the other is
- "http://www.server.com/index.html". Don't remove the old
- one, just add the new one as a non-canonical entry. */
- goto url_only;
+ && !hash_table_contains (dl_url_file_map, url))
+ /* The two URLs differ only in the "index.html" ending. For
+ example, one is "http://www.server.com/", and the other is
+ "http://www.server.com/index.html". Don't remove the old
+ one, just add the new one as a non-canonical entry. */
+ goto url_only;
hash_table_remove (dl_file_url_map, file);
xfree (old_file);
xfree (old_url);
/* Remove all the URLs that point to this file. Yes, there can
- be more than one such URL, because we store redirections as
- multiple entries in dl_url_file_map. For example, if URL1
- redirects to URL2 which gets downloaded to FILE, we map both
- URL1 and URL2 to FILE in dl_url_file_map. (dl_file_url_map
- only points to URL2.) When another URL gets loaded to FILE,
- we want both URL1 and URL2 dissociated from it.
+ be more than one such URL, because we store redirections as
+ multiple entries in dl_url_file_map. For example, if URL1
+ redirects to URL2 which gets downloaded to FILE, we map both
+ URL1 and URL2 to FILE in dl_url_file_map. (dl_file_url_map
+ only points to URL2.) When another URL gets loaded to FILE,
+ we want both URL1 and URL2 dissociated from it.
This is a relatively expensive operation because it performs
a linear search of the whole hash table, but it should be
if (mode == CHECK_FOR_FILE)
{
if (!downloaded_files_hash)
- return FILE_NOT_ALREADY_DOWNLOADED;
+ return FILE_NOT_ALREADY_DOWNLOADED;
ptr = hash_table_get (downloaded_files_hash, file);
if (!ptr)
- return FILE_NOT_ALREADY_DOWNLOADED;
+ return FILE_NOT_ALREADY_DOWNLOADED;
return *ptr;
}
{
hash_table_iterator iter;
for (hash_table_iterate (downloaded_files_hash, &iter);
- hash_table_iter_next (&iter);
- )
- xfree (iter.key);
+ hash_table_iter_next (&iter);
+ )
+ xfree (iter.key);
hash_table_destroy (downloaded_files_hash);
downloaded_files_hash = NULL;
}
for (ptr = list; ptr; ptr = ptr->next)
{
- /* TODO: strcasecmp may not be appropriate to compare URLs */
- if (strcasecmp (url, ptr->url) == 0) return true;
+ /* str[case]cmp is inadequate for URL comparison */
+ if (are_urls_equal (url, ptr->url) == 0) return true;
}
return false;
nonexisting_url (const char *url, const char *referrer)
{
struct broken_urls_list *list;
+
+ /* Ignore robots.txt URLs */
+ if (is_robots_txt_url (url))
+ return;
if (!nonexisting_urls_hash)
nonexisting_urls_hash = make_string_hash_table (0);
{
hash_table_iterator iter;
for (hash_table_iterate (nonexisting_urls_hash, &iter);
- hash_table_iter_next (&iter);
- )
+ hash_table_iter_next (&iter);
+ )
{
- xfree (iter.key);
- xfree (iter.value);
- }
+ xfree (iter.key);
+ xfree (iter.value);
+ }
hash_table_destroy (nonexisting_urls_hash);
nonexisting_urls_hash = NULL;
}
)
{
struct broken_urls_list *list;
-
+
logprintf (LOG_NOTQUIET, _("%s referred by:\n"), (const char *)iter.key);
for (list = (struct broken_urls_list *) iter.value;
list;
- list = list->next)
+ list = list->next)
{
logprintf (LOG_NOTQUIET, _(" %s\n"), list->url);
}
for (i = 0; *s; s++, i++)
{
if (*s == '&')
- i += 4; /* `amp;' */
+ i += 4; /* `amp;' */
else if (*s == '<' || *s == '>')
- i += 3; /* `lt;' and `gt;' */
+ i += 3; /* `lt;' and `gt;' */
else if (*s == '\"')
- i += 5; /* `quot;' */
+ i += 5; /* `quot;' */
else if (*s == ' ')
- i += 4; /* #32; */
+ i += 4; /* #32; */
}
res = xmalloc (i + 1);
s = b;
for (p = res; *s; s++)
{
switch (*s)
- {
- case '&':
- *p++ = '&';
- *p++ = 'a';
- *p++ = 'm';
- *p++ = 'p';
- *p++ = ';';
- break;
- case '<': case '>':
- *p++ = '&';
- *p++ = (*s == '<' ? 'l' : 'g');
- *p++ = 't';
- *p++ = ';';
- break;
- case '\"':
- *p++ = '&';
- *p++ = 'q';
- *p++ = 'u';
- *p++ = 'o';
- *p++ = 't';
- *p++ = ';';
- break;
- case ' ':
- *p++ = '&';
- *p++ = '#';
- *p++ = '3';
- *p++ = '2';
- *p++ = ';';
- break;
- default:
- *p++ = *s;
- }
+ {
+ case '&':
+ *p++ = '&';
+ *p++ = 'a';
+ *p++ = 'm';
+ *p++ = 'p';
+ *p++ = ';';
+ break;
+ case '<': case '>':
+ *p++ = '&';
+ *p++ = (*s == '<' ? 'l' : 'g');
+ *p++ = 't';
+ *p++ = ';';
+ break;
+ case '\"':
+ *p++ = '&';
+ *p++ = 'q';
+ *p++ = 'u';
+ *p++ = 'o';
+ *p++ = 't';
+ *p++ = ';';
+ break;
+ case ' ':
+ *p++ = '&';
+ *p++ = '#';
+ *p++ = '3';
+ *p++ = '2';
+ *p++ = ';';
+ break;
+ default:
+ *p++ = *s;
+ }
}
*p = '\0';
return res;
}
+
+/*
+ * vim: et ts=2 sw=2
+ */
+
projects / wget / blobdiff