[svn] Add FLAGS argument to map_html_tags.

[wget] / src / html-url.c

diff --git a/src/html-url.c b/src/html-url.c
index a3208e468395c33d22325dc5da982a85052b7090..09962eddc9286a0f6a6569c697660caf9f2e96a8 100644 (file)
--- a/src/html-url.c
+++ b/src/html-url.c
@@ -43,6 +43,7 @@ so, delete this exception statement from your version.  */
 #include "html-parse.h"
 #include "url.h"
 #include "utils.h"
+#include "convert.h"
 
 #ifndef errno
 extern int errno;
@@ -269,30 +270,36 @@ init_interesting (void)
   }
 }
 
+/* Find tag with name TAG_NAME in KNOWN_TAGS and return its index.  */
+
 static int
 find_tag (const char *tag_name)
 {
-  int i;
+  /* Originally implemented as linear search.  In Wget 1.9 known_tags
+     contains 21 elements, for which binary search requires max. 5
+     comparisons, whereas linear search performs 10 on average.  */
 
-  /* This is linear search; if the number of tags grow, we can switch
-     to binary search.  */
+  int lo = 0, hi = countof (known_tags) - 1;
 
-  for (i = 0; i < countof (known_tags); i++)
+  while (lo <= hi)
     {
-      int cmp = strcasecmp (known_tags[i].name, tag_name);
-      /* known_tags are sorted alphabetically, so we can
-         micro-optimize.  */
-      if (cmp > 0)
-       break;
-      else if (cmp == 0)
-       return i;
+      int mid = (lo + hi) >> 1;
+      int cmp = strcasecmp (tag_name, known_tags[mid].name);
+      if (cmp < 0)
+       hi = mid - 1;
+      else if (cmp > 0)
+       lo = mid + 1;
+      else
+       return mid;
     }
+
   return -1;
 }
 
 /* Find the value of attribute named NAME in the taginfo TAG.  If the
    attribute is not present, return NULL.  If ATTRIND is non-NULL, the
    index of the attribute in TAG will be stored there.  */
+
 static char *
 find_attr (struct taginfo *tag, const char *name, int *attrind)
 {
@@ -630,11 +637,13 @@ collect_tags_mapper (struct taginfo *tag, void *arg)
 /* Analyze HTML tags FILE and construct a list of URLs referenced from
    it.  It merges relative links in FILE with URL.  It is aware of
    <base href=...> and does the right thing.  */
+
 struct urlpos *
 get_urls_html (const char *file, const char *url, int *meta_disallow_follow)
 {
   struct file_memory *fm;
   struct map_context ctx;
+  int flags;
 
   /* Load the file. */
   fm = read_file (file);
@@ -655,8 +664,16 @@ get_urls_html (const char *file, const char *url, int *meta_disallow_follow)
   if (!interesting_tags)
     init_interesting ();
 
-  map_html_tags (fm->content, fm->length, interesting_tags,
-                interesting_attributes, collect_tags_mapper, &ctx);
+  /* Specify MHT_TRIM_VALUES because of buggy HTML generators that
+     generate <a href=" foo"> instead of <a href="foo"> (Netscape
+     ignores spaces as well.)  If you really mean space, use &32; or
+     %20.  */
+  flags = MHT_TRIM_VALUES;
+  if (opt.strict_comments)
+    flags |= MHT_STRICT_COMMENTS;
+
+  map_html_tags (fm->content, fm->length, collect_tags_mapper, &ctx, flags,
+                interesting_tags, interesting_attributes);
 
   DEBUGP (("no-follow in %s: %d\n", file, ctx.nofollow));
   if (meta_disallow_follow)
@@ -667,6 +684,91 @@ get_urls_html (const char *file, const char *url, int *meta_disallow_follow)
   return ctx.head;
 }
 
+/* This doesn't really have anything to do with HTML, but it's similar
+   to get_urls_html, so we put it here.  */
+
+struct urlpos *
+get_urls_file (const char *file)
+{
+  struct file_memory *fm;
+  struct urlpos *head, *tail;
+  const char *text, *text_end;
+
+  /* Load the file.  */
+  fm = read_file (file);
+  if (!fm)
+    {
+      logprintf (LOG_NOTQUIET, "%s: %s\n", file, strerror (errno));
+      return NULL;
+    }
+  DEBUGP (("Loaded %s (size %ld).\n", file, fm->length));
+
+  head = tail = NULL;
+  text = fm->content;
+  text_end = fm->content + fm->length;
+  while (text < text_end)
+    {
+      int up_error_code;
+      char *url_text;
+      struct urlpos *entry;
+      struct url *url;
+
+      const char *line_beg = text;
+      const char *line_end = memchr (text, '\n', text_end - text);
+      if (!line_end)
+       line_end = text_end;
+      else
+       ++line_end;
+      text = line_end;
+
+      /* Strip whitespace from the beginning and end of line. */
+      while (line_beg < line_end && ISSPACE (*line_beg))
+       ++line_beg;
+      while (line_end > line_beg && ISSPACE (*(line_end - 1)))
+       --line_end;
+
+      if (line_beg == line_end)
+       continue;
+
+      /* The URL is in the [line_beg, line_end) region. */
+
+      /* We must copy the URL to a zero-terminated string, and we
+        can't use alloca because we're in a loop.  *sigh*.  */
+      url_text = strdupdelim (line_beg, line_end);
+
+      if (opt.base_href)
+       {
+         /* Merge opt.base_href with URL. */
+         char *merged = uri_merge (opt.base_href, url_text);
+         xfree (url_text);
+         url_text = merged;
+       }
+
+      url = url_parse (url_text, &up_error_code);
+      if (!url)
+       {
+         logprintf (LOG_NOTQUIET, "%s: Invalid URL %s: %s\n",
+                    file, url_text, url_error (up_error_code));
+         xfree (url_text);
+         continue;
+       }
+      xfree (url_text);
+
+      entry = (struct urlpos *)xmalloc (sizeof (struct urlpos));
+      memset (entry, 0, sizeof (*entry));
+      entry->next = NULL;
+      entry->url = url;
+
+      if (!head)
+       head = entry;
+      else
+       tail->next = entry;
+      tail = entry;
+    }
+  read_file_free (fm);
+  return head;
+}
+
 void
 cleanup_html_url (void)
 {