Merge in gerel's url-parsing stuff.

[wget] / src / recur.c

diff --git a/src/recur.c b/src/recur.c
index c9f038aee1ae294cc505b83aba51ec400bef249e..2e067505c1c6a521c8ebcef66178ad7347fb8184 100644 (file)
--- a/src/recur.c
+++ b/src/recur.c
@@ -154,7 +154,7 @@ url_dequeue (struct url_queue *queue,
 \f
 static bool download_child_p (const struct urlpos *, struct url *, int,
                               struct url *, struct hash_table *);
-static bool descend_redirect_p (const char *, const char *, int,
+static bool descend_redirect_p (const char *, struct url *, int,
                                 struct url *, struct hash_table *);
 
 
@@ -180,7 +180,7 @@ static bool descend_redirect_p (const char *, const char *, int,
           options, add it to the queue. */
 
 uerr_t
-retrieve_tree (const char *start_url)
+retrieve_tree (struct url *start_url_parsed)
 {
   uerr_t status = RETROK;
 
@@ -191,16 +191,6 @@ retrieve_tree (const char *start_url)
      the queue, but haven't been downloaded yet.  */
   struct hash_table *blacklist;
 
-  int up_error_code;
-  struct url *start_url_parsed = url_parse (start_url, &up_error_code);
-
-  if (!start_url_parsed)
-    {
-      logprintf (LOG_NOTQUIET, "%s: %s.\n", start_url,
-                 url_error (up_error_code));
-      return URLERROR;
-    }
-
   queue = url_queue_new ();
   blacklist = make_string_hash_table (0);
 
@@ -263,10 +253,22 @@ retrieve_tree (const char *start_url)
         }
       else
         {
-          int dt = 0;
+          int dt = 0, url_err;
           char *redirected = NULL;
+          struct url *url_parsed = url_parse (url, &url_err);
 
-          status = retrieve_url (url, &file, &redirected, referer, &dt, false);
+          if (!url_parsed)
+            {
+              char *error = url_error (url, url_err);
+              logprintf (LOG_NOTQUIET, "%s: %s.\n", url, error);
+              xfree (error);
+              status = URLERROR;
+            }
+          else
+            {
+              status = retrieve_url (url_parsed, url, &file, &redirected,
+                                     referer, &dt, false);
+            }
 
           if (html_allowed && file && status == RETROK
               && (dt & RETROKF) && (dt & TEXTHTML))
@@ -293,7 +295,7 @@ retrieve_tree (const char *start_url)
                  want to follow it.  */
               if (descend)
                 {
-                  if (!descend_redirect_p (redirected, url, depth,
+                  if (!descend_redirect_p (redirected, url_parsed, depth,
                                            start_url_parsed, blacklist))
                     descend = false;
                   else
@@ -305,6 +307,7 @@ retrieve_tree (const char *start_url)
               xfree (url);
               url = redirected;
             }
+          url_free(url_parsed);
         }
 
       if (opt.spider)
@@ -438,8 +441,6 @@ retrieve_tree (const char *start_url)
   }
   url_queue_delete (queue);
 
-  if (start_url_parsed)
-    url_free (start_url_parsed);
   string_set_free (blacklist);
 
   if (opt.quota && total_downloaded_bytes > opt.quota)
@@ -473,7 +474,7 @@ download_child_p (const struct urlpos *upos, struct url *parent, int depth,
       if (opt.spider) 
         {
           char *referrer = url_string (parent, URL_AUTH_HIDE_PASSWD);
-          DEBUGP (("download_child_p: parent->url is: `%s'\n", parent->url));
+          DEBUGP (("download_child_p: parent->url is: %s\n", quote (parent->url)));
           visited_url (url, referrer);
           xfree (referrer);
         }
@@ -655,14 +656,13 @@ download_child_p (const struct urlpos *upos, struct url *parent, int depth,
    it is merely a simple-minded wrapper around download_child_p.  */
 
 static bool
-descend_redirect_p (const char *redirected, const char *original, int depth,
+descend_redirect_p (const char *redirected, struct url *orig_parsed, int depth,
                     struct url *start_url_parsed, struct hash_table *blacklist)
 {
-  struct url *orig_parsed, *new_parsed;
+  struct url *new_parsed;
   struct urlpos *upos;
   bool success;
 
-  orig_parsed = url_parse (original, NULL);
   assert (orig_parsed != NULL);
 
   new_parsed = url_parse (redirected, NULL);
@@ -674,7 +674,6 @@ descend_redirect_p (const char *redirected, const char *original, int depth,
   success = download_child_p (upos, orig_parsed, depth,
                               start_url_parsed, blacklist);
 
-  url_free (orig_parsed);
   url_free (new_parsed);
   xfree (upos);