X-Git-Url: http://sjero.net/git/?a=blobdiff_plain;f=src%2Fhtml-url.c;h=c7c3e39ab55929e2f23028ddc25046575c468515;hb=0db305126bd05f0739d83d36634772cf44851a65;hp=ef93a7e49c5c61ded555c2b6c022be3346bbec2c;hpb=b30a0dd817886f77a64be9218c5e5399bcbc2e67;p=wget
diff --git a/src/html-url.c b/src/html-url.c
index ef93a7e4..c7c3e39a 100644
--- a/src/html-url.c
+++ b/src/html-url.c
@@ -1,6 +1,6 @@
/* Collect URLs from HTML source.
Copyright (C) 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
- 2007, 2008 Free Software Foundation, Inc.
+ 2007, 2008, 2009, 2010, 2011 Free Software Foundation, Inc.
This file is part of GNU Wget.
@@ -44,7 +44,6 @@ as that of the covered work. */
#include "recur.h"
#include "html-url.h"
#include "css-url.h"
-#include "iri.h"
typedef void (*tag_handler_t) (int, struct taginfo *, struct map_context *);
@@ -165,6 +164,7 @@ static struct {
to the attributes not mentioned here. We add them manually. */
static const char *additional_attributes[] = {
"rel", /* used by tag_handle_link */
+ "type", /* used by tag_handle_link */
"http-equiv", /* used by tag_handle_meta */
"name", /* used by tag_handle_meta */
"content", /* used by tag_handle_meta */
@@ -175,6 +175,10 @@ static const char *additional_attributes[] = {
static struct hash_table *interesting_tags;
static struct hash_table *interesting_attributes;
+/* Will contains the (last) charset found in 'http-equiv=content-type'
+ meta tags */
+static char *meta_charset;
+
static void
init_interesting (void)
{
@@ -285,9 +289,7 @@ append_url (const char *link_uri, int position, int size,
return NULL;
}
- set_ugly_no_encode (true);
- url = url_parse (link_uri, NULL);
- set_ugly_no_encode (false);
+ url = url_parse (link_uri, NULL, NULL, false);
if (!url)
{
DEBUGP (("%s: link \"%s\" doesn't parse.\n",
@@ -303,12 +305,13 @@ append_url (const char *link_uri, int position, int size,
char *complete_uri = uri_merge (base, link_uri);
- DEBUGP (("%s: merge(\"%s\", \"%s\") -> %s\n",
- ctx->document_file, base, link_uri, complete_uri));
+ DEBUGP (("%s: merge(%s, %s) -> %s\n",
+ quotearg_n_style (0, escape_quoting_style, ctx->document_file),
+ quote_n (1, base),
+ quote_n (2, link_uri),
+ quotearg_n_style (3, escape_quoting_style, complete_uri)));
- set_ugly_no_encode (true);
- url = url_parse (complete_uri, NULL);
- set_ugly_no_encode (false);
+ url = url_parse (complete_uri, NULL, NULL, false);
if (!url)
{
DEBUGP (("%s: merged link \"%s\" doesn't parse.\n",
@@ -319,7 +322,7 @@ append_url (const char *link_uri, int position, int size,
xfree (complete_uri);
}
- DEBUGP (("appending \"%s\" to urlpos.\n", url->url));
+ DEBUGP (("appending %s to urlpos.\n", quote (url->url)));
newel = xnew0 (struct urlpos);
newel->url = url;
@@ -333,13 +336,27 @@ append_url (const char *link_uri, int position, int size,
else if (link_has_scheme)
newel->link_complete_p = 1;
- if (ctx->tail)
+ /* Append the new URL maintaining the order by position. */
+ if (ctx->head == NULL)
+ ctx->head = newel;
+ else
{
- ctx->tail->next = newel;
- ctx->tail = newel;
+ struct urlpos *it, *prev = NULL;
+
+ it = ctx->head;
+ while (it && position > it->pos)
+ {
+ prev = it;
+ it = it->next;
+ }
+
+ newel->next = it;
+
+ if (prev)
+ prev->next = newel;
+ else
+ ctx->head = newel;
}
- else
- ctx->tail = ctx->head = newel;
return newel;
}
@@ -348,12 +365,27 @@ static void
check_style_attr (struct taginfo *tag, struct map_context *ctx)
{
int attrind;
+ int raw_start;
+ int raw_len;
char *style = find_attr (tag, "style", &attrind);
if (!style)
return;
- /* raw pos and raw size include the quotes, hence the +1 -2 */
- get_urls_css (ctx, ATTR_POS(tag,attrind,ctx)+1, ATTR_SIZE(tag,attrind)-2);
+ /* raw pos and raw size include the quotes, skip them when they are
+ present. */
+ raw_start = ATTR_POS (tag, attrind, ctx);
+ raw_len = ATTR_SIZE (tag, attrind);
+ if( *(char *)(ctx->text + raw_start) == '\''
+ || *(char *)(ctx->text + raw_start) == '"')
+ {
+ raw_start += 1;
+ raw_len -= 2;
+ }
+
+ if(raw_len <= 0)
+ return;
+
+ get_urls_css (ctx, raw_start, raw_len);
}
/* All the tag_* functions are called from collect_tags_mapper, as
@@ -474,8 +506,8 @@ tag_handle_link (int tagid, struct taginfo *tag, struct map_context *ctx)
/* All link references are external, except those
known not to be, such as style sheet and shortcut icon:
-
-
+
+
*/
if (href)
{
@@ -495,11 +527,18 @@ tag_handle_link (int tagid, struct taginfo *tag, struct map_context *ctx)
{
up->link_inline_p = 1;
}
+ else
+ {
+ /* The external ones usually point to HTML pages, such as
+
+ except when the type attribute says otherwise:
+
+ */
+ char *type = find_attr (tag, "type", NULL);
+ if (!type || strcasecmp (type, "text/html") == 0)
+ up->link_expect_html = 1;
+ }
}
- else
- /* The external ones usually point to HTML pages, such as
- */
- up->link_expect_html = 1;
}
}
}
@@ -572,10 +611,8 @@ tag_handle_meta (int tagid, struct taginfo *tag, struct map_context *ctx)
if (!mcharset)
return;
- /*logprintf (LOG_VERBOSE, "Meta tag charset : %s\n", quote (mcharset));*/
-
- set_current_charset (mcharset);
- xfree (mcharset);
+ xfree_null (meta_charset);
+ meta_charset = mcharset;
}
else if (name && 0 == strcasecmp (name, "robots"))
{
@@ -590,15 +627,25 @@ tag_handle_meta (int tagid, struct taginfo *tag, struct map_context *ctx)
{
while (*content)
{
- /* Find the next occurrence of ',' or the end of
- the string. */
- char *end = strchr (content, ',');
- if (end)
- ++end;
- else
- end = content + strlen (content);
+ char *end;
+ /* Skip any initial whitespace. */
+ content += strspn (content, " \f\n\r\t\v");
+ /* Find the next occurrence of ',' or whitespace,
+ * or the end of the string. */
+ end = content + strcspn (content, ", \f\n\r\t\v");
if (!strncasecmp (content, "nofollow", end - content))
ctx->nofollow = true;
+ /* Skip past the next comma, if any. */
+ if (*end == ',')
+ ++end;
+ else
+ {
+ end = strchr (end, ',');
+ if (end)
+ ++end;
+ else
+ end = content + strlen (content);
+ }
content = end;
}
}
@@ -615,7 +662,7 @@ collect_tags_mapper (struct taginfo *tag, void *arg)
/* Find the tag in our table of tags. This must not fail because
map_html_tags only returns tags found in interesting_tags.
-
+
I've changed this for now, I'm passing NULL as interesting_tags
to map_html_tags. This way we can check all tags for a style
attribute.
@@ -641,14 +688,15 @@ collect_tags_mapper (struct taginfo *tag, void *arg)
and does the right thing. */
struct urlpos *
-get_urls_html (const char *file, const char *url, bool *meta_disallow_follow)
+get_urls_html (const char *file, const char *url, bool *meta_disallow_follow,
+ struct iri *iri)
{
struct file_memory *fm;
struct map_context ctx;
int flags;
/* Load the file. */
- fm = read_file (file);
+ fm = wget_read_file (file);
if (!fm)
{
logprintf (LOG_NOTQUIET, "%s: %s\n", file, strerror (errno));
@@ -657,7 +705,7 @@ get_urls_html (const char *file, const char *url, bool *meta_disallow_follow)
DEBUGP (("Loaded %s (size %s).\n", file, number_to_static_string (fm->length)));
ctx.text = fm->content;
- ctx.head = ctx.tail = NULL;
+ ctx.head = NULL;
ctx.base = NULL;
ctx.parent_base = url ? url : opt.base_href;
ctx.document_file = file;
@@ -681,12 +729,16 @@ get_urls_html (const char *file, const char *url, bool *meta_disallow_follow)
map_html_tags (fm->content, fm->length, collect_tags_mapper, &ctx, flags,
NULL, interesting_attributes);
+ /* If meta charset isn't null, override content encoding */
+ if (iri && meta_charset)
+ set_content_encoding (iri, meta_charset);
+
DEBUGP (("no-follow in %s: %d\n", file, ctx.nofollow));
if (meta_disallow_follow)
*meta_disallow_follow = ctx.nofollow;
xfree_null (ctx.base);
- read_file_free (fm);
+ wget_read_file_free (fm);
return ctx.head;
}
@@ -701,7 +753,7 @@ get_urls_file (const char *file)
const char *text, *text_end;
/* Load the file. */
- fm = read_file (file);
+ fm = wget_read_file (file);
if (!fm)
{
logprintf (LOG_NOTQUIET, "%s: %s\n", file, strerror (errno));
@@ -750,14 +802,15 @@ get_urls_file (const char *file)
url_text = merged;
}
- set_ugly_no_encode (true);
- url = url_parse (url_text, &up_error_code);
- set_ugly_no_encode (false);
+ url = url_parse (url_text, &up_error_code, NULL, false);
if (!url)
{
+ char *error = url_error (url_text, up_error_code);
logprintf (LOG_NOTQUIET, _("%s: Invalid URL %s: %s\n"),
- file, url_text, url_error (up_error_code));
+ file, url_text, error);
xfree (url_text);
+ xfree (error);
+ inform_exit_status (URLERROR);
continue;
}
xfree (url_text);
@@ -771,7 +824,7 @@ get_urls_file (const char *file)
tail->next = entry;
tail = entry;
}
- read_file_free (fm);
+ wget_read_file_free (fm);
return head;
}