[svn] Several fixes for recursive spider mode.

[wget] / src / convert.c

diff --git a/src/convert.c b/src/convert.c
index 7def7c89195ebedd88fe688def904650cfb8c58d..a59e3c516cf7709c3c72f66f5c2a3548b15f8e3a 100644 (file)
--- a/src/convert.c
+++ b/src/convert.c
@@ -54,8 +54,6 @@ struct hash_table *dl_url_file_map;
    conversion after Wget is done.  */
 struct hash_table *downloaded_html_set;
 
-static struct hash_table *nonexisting_urls_hash;
-
 static void convert_links (const char *, struct urlpos *);
 
 /* This function is called when the retrieval is done to convert the
@@ -835,7 +833,6 @@ register_html (const char *url, const char *file)
 }
 
 static void downloaded_files_free (void);
-static void nonexisting_urls_free (void);
 
 /* Cleanup the data structures associated with this file.  */
 
@@ -857,7 +854,6 @@ convert_cleanup (void)
   if (downloaded_html_set)
     string_set_free (downloaded_html_set);
   downloaded_files_free ();
-  nonexisting_urls_free ();
   if (converted_files)
     string_set_free (converted_files);
 }
@@ -957,122 +953,6 @@ downloaded_files_free (void)
       downloaded_files_hash = NULL;
     }
 }
-\f
-/* Remembers broken links.  */
-
-struct broken_urls_list 
-{
-  char *url;
-  struct broken_urls_list *next;
-};
-
-static bool
-in_list (const struct broken_urls_list *list, const char *url)
-{
-  const struct broken_urls_list *ptr;
-  
-  for (ptr = list; ptr; ptr = ptr->next)
-    {
-      /* str[case]cmp is inadequate for URL comparison */
-      if (are_urls_equal (url, ptr->url) == 0) return true;      
-    }
-  
-  return false;
-}
-
-void
-nonexisting_url (const char *url, const char *referrer)
-{
-  struct broken_urls_list *list;
- 
-  /* Ignore robots.txt URLs */
-  if (is_robots_txt_url (url))
-    return;
-  
-  if (!nonexisting_urls_hash)
-    nonexisting_urls_hash = make_string_hash_table (0);
-
-  list = hash_table_get (nonexisting_urls_hash, url);
-  if (!list)
-    {
-      list = (struct broken_urls_list *) xnew0 (struct broken_urls_list);
-      list->url = referrer ? xstrdup (referrer) : NULL;
-      hash_table_put (nonexisting_urls_hash, xstrdup (url), list);
-    }
-  else if (list && !in_list (list, referrer)) 
-    {
-      /* Append referrer at the end of the list */
-      struct broken_urls_list *newnode;
-      
-      while (list->next) list = list->next;
-      
-      newnode = xnew0 (struct broken_urls_list);
-      newnode->url = xstrdup (referrer);
-      list->next = newnode;
-    }
-}
-
-static void
-nonexisting_urls_free (void)
-{
-  if (nonexisting_urls_hash)
-    {
-      hash_table_iterator iter;
-      for (hash_table_iterate (nonexisting_urls_hash, &iter);
-           hash_table_iter_next (&iter);
-           )
-        {
-          xfree (iter.key);
-          xfree (iter.value);
-        }
-      hash_table_destroy (nonexisting_urls_hash);
-      nonexisting_urls_hash = NULL;
-    }
-}
-
-void
-print_broken_links (void)
-{
-  hash_table_iterator iter;
-  int num_elems;
-  
-  if (!nonexisting_urls_hash) 
-    {
-      logprintf (LOG_NOTQUIET, _("Found no broken links.\n\n"));
-      return;
-    }
-  
-  num_elems = hash_table_count (nonexisting_urls_hash);
-  assert (num_elems > 0);
-  
-  if (num_elems > 1) 
-    {
-      logprintf (LOG_NOTQUIET, _("Found %d broken links.\n\n"), 
-                 num_elems);
-    }
-  else
-    {
-      logprintf (LOG_NOTQUIET, _("Found 1 broken link.\n\n"));
-    }
-  
-  for (hash_table_iterate (nonexisting_urls_hash, &iter);
-       hash_table_iter_next (&iter);
-       )
-    {
-      struct broken_urls_list *list;
-          
-      logprintf (LOG_NOTQUIET, _("%s referred by:\n"), (const char *)iter.key);
-
-      for (list = (struct broken_urls_list *) iter.value; 
-           list; 
-           list = list->next) 
-        {
-          logprintf (LOG_NOTQUIET, _("    %s\n"), list->url);
-        }
-    }
-  logputs (LOG_NOTQUIET, "\n");
-}
-
 \f
 /* The function returns the pointer to the malloc-ed quoted version of
    string s.  It will recognize and quote numeric and special graphic