Improved exit status handling.

[wget] / src / recur.c

diff --git a/src/recur.c b/src/recur.c
index aa83e9a6c15651a03a0961d7b8a7be9f6e33d688..66ef2e0fd9b566d0c9c809562daa2d4ee32ddb70 100644 (file)
--- a/src/recur.c
+++ b/src/recur.c
@@ -51,7 +51,6 @@ as that of the covered work.  */
 #include "html-url.h"
 #include "css-url.h"
 #include "spider.h"
-#include "iri.h"
 \f
 /* Functions for maintaining the URL queue.  */
 
@@ -112,12 +111,13 @@ url_enqueue (struct url_queue *queue, struct iri *i,
   if (queue->count > queue->maxcount)
     queue->maxcount = queue->count;
 
-  DEBUGP (("Enqueuing %s at depth %d\n", url, depth));
+  DEBUGP (("Enqueuing %s at depth %d\n",
+           quotearg_n_style (0, escape_quoting_style, url), depth));
   DEBUGP (("Queue count %d, maxcount %d.\n", queue->count, queue->maxcount));
 
   if (i)
-    DEBUGP (("[IRI Enqueuing %s with %s\n", quote (url),
-               quote (i->uri_encoding)));
+    DEBUGP (("[IRI Enqueuing %s with %s\n", quote_n (0, url),
+             i->uri_encoding ? quote_n (1, i->uri_encoding) : "None"));
 
   if (queue->tail)
     queue->tail->next = qel;
@@ -153,7 +153,8 @@ url_dequeue (struct url_queue *queue, struct iri **i,
 
   --queue->count;
 
-  DEBUGP (("Dequeuing %s at depth %d\n", qel->url, qel->depth));
+  DEBUGP (("Dequeuing %s at depth %d\n",
+           quotearg_n_style (0, escape_quoting_style, qel->url), qel->depth));
   DEBUGP (("Queue count %d, maxcount %d.\n", queue->count, queue->maxcount));
 
   xfree (qel);
@@ -162,7 +163,7 @@ url_dequeue (struct url_queue *queue, struct iri **i,
 \f
 static bool download_child_p (const struct urlpos *, struct url *, int,
                               struct url *, struct hash_table *, struct iri *);
-static bool descend_redirect_p (const char *, const char *, int,
+static bool descend_redirect_p (const char *, struct url *, int,
                                 struct url *, struct hash_table *, struct iri *);
 
 
@@ -188,7 +189,7 @@ static bool descend_redirect_p (const char *, const char *, int,
           options, add it to the queue. */
 
 uerr_t
-retrieve_tree (const char *start_url)
+retrieve_tree (struct url *start_url_parsed, struct iri *pi)
 {
   uerr_t status = RETROK;
 
@@ -200,17 +201,19 @@ retrieve_tree (const char *start_url)
   struct hash_table *blacklist;
 
   int up_error_code;
-  struct url *start_url_parsed;
   struct iri *i = iri_new ();
-  set_uri_encoding (i, opt.locale);
 
-  start_url_parsed = url_parse (start_url, &up_error_code, i);
-  if (!start_url_parsed)
+#define COPYSTR(x)  (x) ? xstrdup(x) : NULL;
+  /* Duplicate pi struct if not NULL */
+  if (pi)
     {
-      logprintf (LOG_NOTQUIET, "%s: %s.\n", start_url,
-                 url_error (up_error_code));
-      return URLERROR;
+      i->uri_encoding = COPYSTR (pi->uri_encoding);
+      i->content_encoding = COPYSTR (pi->content_encoding);
+      i->utf8_encode = pi->utf8_encode;
     }
+  else
+    set_uri_encoding (i, opt.locale, true);
+#undef COPYSTR
 
   queue = url_queue_new ();
   blacklist = make_string_hash_table (0);
@@ -275,11 +278,12 @@ retrieve_tree (const char *start_url)
         }
       else
         {
-          int dt = 0;
+          int dt = 0, url_err;
           char *redirected = NULL;
+          struct url *url_parsed = url_parse (url, &url_err, i, true);
 
-          status = retrieve_url (url, &file, &redirected, referer, &dt,
-                                 false, i);
+          status = retrieve_url (url_parsed, url, &file, &redirected, referer,
+                                 &dt, false, i, true);
 
           if (html_allowed && file && status == RETROK
               && (dt & RETROKF) && (dt & TEXTHTML))
@@ -306,7 +310,7 @@ retrieve_tree (const char *start_url)
                  want to follow it.  */
               if (descend)
                 {
-                  if (!descend_redirect_p (redirected, url, depth,
+                  if (!descend_redirect_p (redirected, url_parsed, depth,
                                            start_url_parsed, blacklist, i))
                     descend = false;
                   else
@@ -318,6 +322,12 @@ retrieve_tree (const char *start_url)
               xfree (url);
               url = redirected;
             }
+          else
+            {
+              xfree (url);
+              url = xstrdup (url_parsed->url);
+            }
+          url_free(url_parsed);
         }
 
       if (opt.spider)
@@ -370,7 +380,7 @@ retrieve_tree (const char *start_url)
           if (children)
             {
               struct urlpos *child = children;
-              struct url *url_parsed = url_parse (url, NULL, i);
+              struct url *url_parsed = url_parse (url, NULL, i, true);
               struct iri *ci;
               char *referer_url = url;
               bool strip_auth = (url_parsed != NULL
@@ -391,7 +401,7 @@ retrieve_tree (const char *start_url)
                                         blacklist, i))
                     {
                       ci = iri_new ();
-                      set_uri_encoding (ci, i->content_encoding);
+                      set_uri_encoding (ci, i->content_encoding, false);
                       url_enqueue (queue, ci, xstrdup (child->url->url),
                                    xstrdup (referer_url), depth + 1,
                                    child->link_expect_html,
@@ -457,8 +467,6 @@ retrieve_tree (const char *start_url)
   }
   url_queue_delete (queue);
 
-  if (start_url_parsed)
-    url_free (start_url_parsed);
   string_set_free (blacklist);
 
   if (opt.quota && total_downloaded_bytes > opt.quota)
@@ -675,18 +683,17 @@ download_child_p (const struct urlpos *upos, struct url *parent, int depth,
    it is merely a simple-minded wrapper around download_child_p.  */
 
 static bool
-descend_redirect_p (const char *redirected, const char *original, int depth,
+descend_redirect_p (const char *redirected, struct url *orig_parsed, int depth,
                     struct url *start_url_parsed, struct hash_table *blacklist,
                     struct iri *iri)
 {
-  struct url *orig_parsed, *new_parsed;
+  struct url *new_parsed;
   struct urlpos *upos;
   bool success;
 
-  orig_parsed = url_parse (original, NULL, NULL);
   assert (orig_parsed != NULL);
 
-  new_parsed = url_parse (redirected, NULL, NULL);
+  new_parsed = url_parse (redirected, NULL, NULL, false);
   assert (new_parsed != NULL);
 
   upos = xnew0 (struct urlpos);
@@ -695,7 +702,6 @@ descend_redirect_p (const char *redirected, const char *original, int depth,
   success = download_child_p (upos, orig_parsed, depth,
                               start_url_parsed, blacklist, iri);
 
-  url_free (orig_parsed);
   url_free (new_parsed);
   xfree (upos);