#include "utils.h"
#include "hash.h"
#include "convert.h"
+#include "recur.h" /* declaration of get_urls_html */
#ifndef errno
extern int errno;
TAG_LAYER,
TAG_LINK,
TAG_META,
+ TAG_OBJECT,
TAG_OVERLAY,
TAG_SCRIPT,
TAG_TABLE,
{ TAG_LAYER, "layer", tag_find_urls },
{ TAG_LINK, "link", tag_handle_link },
{ TAG_META, "meta", tag_handle_meta },
+ { TAG_OBJECT, "object", tag_find_urls },
{ TAG_OVERLAY, "overlay", tag_find_urls },
{ TAG_SCRIPT, "script", tag_find_urls },
{ TAG_TABLE, "table", tag_find_urls },
{ TAG_IMG, "src", ATTR_INLINE },
{ TAG_INPUT, "src", ATTR_INLINE },
{ TAG_LAYER, "src", ATTR_INLINE | ATTR_HTML },
+ { TAG_OBJECT, "data", ATTR_INLINE },
{ TAG_OVERLAY, "src", ATTR_INLINE | ATTR_HTML },
{ TAG_SCRIPT, "src", ATTR_INLINE },
{ TAG_TABLE, "background", ATTR_INLINE },
/* Add the attributes we care about. */
interesting_attributes = make_nocase_string_hash_table (10);
for (i = 0; i < countof (additional_attributes); i++)
- string_set_add (interesting_attributes, additional_attributes[i]);
+ hash_table_put (interesting_attributes, additional_attributes[i], "1");
for (i = 0; i < countof (tag_url_attributes); i++)
- string_set_add (interesting_attributes, tag_url_attributes[i].attr_name);
+ hash_table_put (interesting_attributes,
+ tag_url_attributes[i].attr_name, "1");
}
/* Find the value of attribute named NAME in the taginfo TAG. If the
size. */
static struct urlpos *
-append_one_url (const char *link_uri,
- struct taginfo *tag, int attrind, struct map_context *ctx)
+append_url (const char *link_uri,
+ struct taginfo *tag, int attrind, struct map_context *ctx)
{
int link_has_scheme = url_has_scheme (link_uri);
struct urlpos *newel;
DEBUGP (("appending \"%s\" to urlpos.\n", url->url));
- newel = (struct urlpos *)xmalloc (sizeof (struct urlpos));
- memset (newel, 0, sizeof (*newel));
-
- newel->next = NULL;
+ newel = xnew0 (struct urlpos);
newel->url = url;
newel->pos = tag->attrs[attrind].value_raw_beginning - ctx->text;
newel->size = tag->attrs[attrind].value_raw_size;
if (0 == strcasecmp (tag->attrs[attrind].name,
tag_url_attributes[i].attr_name))
{
- struct urlpos *up = append_one_url (link, tag, attrind, ctx);
+ struct urlpos *up = append_url (link, tag, attrind, ctx);
if (up)
{
int flags = tag_url_attributes[i].flags;
if (!newbase)
return;
- base_urlpos = append_one_url (newbase, tag, attrind, ctx);
+ base_urlpos = append_url (newbase, tag, attrind, ctx);
if (!base_urlpos)
return;
base_urlpos->ignore_when_downloading = 1;
char *action = find_attr (tag, "action", &attrind);
if (action)
{
- struct urlpos *up = append_one_url (action, tag, attrind, ctx);
+ struct urlpos *up = append_url (action, tag, attrind, ctx);
if (up)
up->ignore_when_downloading = 1;
}
*/
if (href)
{
- struct urlpos *up = append_one_url (href, tag, attrind, ctx);
+ struct urlpos *up = append_url (href, tag, attrind, ctx);
if (up)
{
char *rel = find_attr (tag, "rel", NULL);
&& (0 == strcasecmp (rel, "stylesheet")
|| 0 == strcasecmp (rel, "shortcut icon")))
up->link_inline_p = 1;
+ else
+ /* The external ones usually point to HTML pages, such as
+ <link rel="next" href="..."> */
+ up->link_expect_html = 1;
}
}
}
while (ISSPACE (*p))
++p;
- entry = append_one_url (p, tag, attrind, ctx);
+ entry = append_url (p, tag, attrind, ctx);
if (entry)
{
entry->link_refresh_p = 1;
entry->refresh_timeout = timeout;
+ entry->link_expect_html = 1;
}
}
else if (name && 0 == strcasecmp (name, "robots"))
logprintf (LOG_NOTQUIET, "%s: %s\n", file, strerror (errno));
return NULL;
}
- DEBUGP (("Loaded %s (size %ld).\n", file, fm->length));
+ DEBUGP (("Loaded %s (size %s).\n", file, number_to_static_string (fm->length)));
ctx.text = fm->content;
ctx.head = ctx.tail = NULL;
init_interesting ();
/* Specify MHT_TRIM_VALUES because of buggy HTML generators that
- generate <a href=" foo"> instead of <a href="foo"> (Netscape
- ignores spaces as well.) If you really mean space, use &32; or
- %20. */
+ generate <a href=" foo"> instead of <a href="foo"> (browsers
+ ignore spaces as well.) If you really mean space, use &32; or
+ %20. MHT_TRIM_VALUES also causes squashing of embedded newlines,
+ e.g. in <img src="foo.[newline]html">. Such newlines are also
+ ignored by IE and Mozilla and are presumably introduced by
+ writing HTML with editors that force word wrap. */
flags = MHT_TRIM_VALUES;
if (opt.strict_comments)
flags |= MHT_STRICT_COMMENTS;
if (meta_disallow_follow)
*meta_disallow_follow = ctx.nofollow;
- FREE_MAYBE (ctx.base);
+ xfree_null (ctx.base);
read_file_free (fm);
return ctx.head;
}
logprintf (LOG_NOTQUIET, "%s: %s\n", file, strerror (errno));
return NULL;
}
- DEBUGP (("Loaded %s (size %ld).\n", file, fm->length));
+ DEBUGP (("Loaded %s (size %s).\n", file, number_to_static_string (fm->length)));
head = tail = NULL;
text = fm->content;
url = url_parse (url_text, &up_error_code);
if (!url)
{
- logprintf (LOG_NOTQUIET, "%s: Invalid URL %s: %s\n",
+ logprintf (LOG_NOTQUIET, _("%s: Invalid URL %s: %s\n"),
file, url_text, url_error (up_error_code));
xfree (url_text);
continue;
}
xfree (url_text);
- entry = (struct urlpos *)xmalloc (sizeof (struct urlpos));
- memset (entry, 0, sizeof (*entry));
+ entry = xnew0 (struct urlpos);
entry->next = NULL;
entry->url = url;
void
cleanup_html_url (void)
{
- FREE_MAYBE (interesting_tags);
- FREE_MAYBE (interesting_attributes);
+ /* Destroy the hash tables. The hash table keys and values are not
+ allocated by this code, so we don't need to free them here. */
+ if (interesting_tags)
+ hash_table_destroy (interesting_tags);
+ if (interesting_attributes)
+ hash_table_destroy (interesting_attributes);
}
projects / wget / blobdiff