Automated merge.

[wget] / src / retr.c

diff --git a/src/retr.c b/src/retr.c
index 857742979e397748f3932d566d77ecf0d89dcd7b..c79410ff893aa406a77cc3a10bb750464c8c0d3b 100644 (file)
--- a/src/retr.c
+++ b/src/retr.c
@@ -597,7 +597,7 @@ static char *getproxy (struct url *);
 
 uerr_t
 retrieve_url (const char *origurl, char **file, char **newloc,
-              const char *refurl, int *dt, bool recursive)
+              const char *refurl, int *dt, bool recursive, struct iri *iri)
 {
   uerr_t result;
   char *url;
@@ -625,7 +625,8 @@ retrieve_url (const char *origurl, char **file, char **newloc,
   if (file)
     *file = NULL;
 
-  u = url_parse (url, &up_error_code);
+ second_try:
+  u = url_parse (url, &up_error_code, iri);
   if (!u)
     {
       char *error = url_error (url, up_error_code);
@@ -635,6 +636,10 @@ retrieve_url (const char *origurl, char **file, char **newloc,
       return URLERROR;
     }
 
+  DEBUGP (("[IRI Retrieving %s with %s (UTF-8=%d)\n", quote (url),
+           iri->uri_encoding ? quote (iri->uri_encoding) : "None",
+           iri->utf8_encode));
+
   if (!refurl)
     refurl = opt.referer;
 
@@ -648,8 +653,13 @@ retrieve_url (const char *origurl, char **file, char **newloc,
   proxy = getproxy (u);
   if (proxy)
     {
+      /* sXXXav : could a proxy include a path ??? */
+      struct iri *pi = iri_new ();
+      set_uri_encoding (pi, opt.locale, true);
+      pi->utf8_encode = false;
+
       /* Parse the proxy URL.  */
-      proxy_url = url_parse (proxy, &up_error_code);
+      proxy_url = url_parse (proxy, &up_error_code, NULL);
       if (!proxy_url)
         {
           char *error = url_error (proxy, up_error_code);
@@ -676,7 +686,7 @@ retrieve_url (const char *origurl, char **file, char **newloc,
 #endif
       || (proxy_url && proxy_url->scheme == SCHEME_HTTP))
     {
-      result = http_loop (u, &mynewloc, &local_file, refurl, dt, proxy_url);
+      result = http_loop (u, &mynewloc, &local_file, refurl, dt, proxy_url, iri);
     }
   else if (u->scheme == SCHEME_FTP)
     {
@@ -726,8 +736,13 @@ retrieve_url (const char *origurl, char **file, char **newloc,
       xfree (mynewloc);
       mynewloc = construced_newloc;
 
+      /* Reset UTF-8 encoding state, keep the URI encoding and reset
+         the content encoding. */
+      iri->utf8_encode = opt.enable_iri;
+      set_content_encoding (iri, NULL);
+
       /* Now, see if this new location makes sense. */
-      newloc_parsed = url_parse (mynewloc, &up_error_code);
+      newloc_parsed = url_parse (mynewloc, &up_error_code, iri);
       if (!newloc_parsed)
         {
           char *error = url_error (mynewloc, up_error_code);
@@ -776,8 +791,21 @@ retrieve_url (const char *origurl, char **file, char **newloc,
       goto redirected;
     }
 
-  if (local_file)
+  /* Try to not encode in UTF-8 if fetching failed */
+  if (!(*dt & RETROKF) && iri->utf8_encode)
     {
+      iri->utf8_encode = false;
+      DEBUGP (("[IRI Fallbacking to non-utf8 for %s\n", quote (url)));
+      goto second_try;
+    }
+
+  if (local_file && *dt & RETROKF)
+    {
+      register_download (u->url, local_file);
+      if (redirection_count && 0 != strcmp (origurl, u->url))
+        register_redirection (origurl, u->url);
+      if (*dt & TEXTHTML)
+        register_html (u->url, local_file);
       if (*dt & RETROKF)
         {
           register_download (u->url, local_file);
@@ -827,13 +855,17 @@ retrieve_from_file (const char *file, bool html, int *count)
 {
   uerr_t status;
   struct urlpos *url_list, *cur_url;
+  struct iri *iri = iri_new();
 
   char *input_file = NULL;
   const char *url = file;
 
   status = RETROK;             /* Suppose everything is OK.  */
   *count = 0;                  /* Reset the URL count.  */
-  
+
+  /* sXXXav : Assume filename and links in the file are in the locale */
+  set_content_encoding (iri, opt.locale);
+
   if (url_has_scheme (url))
     {
       int dt;
@@ -842,7 +874,7 @@ retrieve_from_file (const char *file, bool html, int *count)
       if (!opt.base_href)
         opt.base_href = xstrdup (url);
 
-      status = retrieve_url (url, &input_file, NULL, NULL, &dt, false);
+      status = retrieve_url (url, &input_file, NULL, NULL, &dt, false, iri);
       if (status != RETROK)
         return status;
 
@@ -852,7 +884,7 @@ retrieve_from_file (const char *file, bool html, int *count)
   else
     input_file = (char *) file;
 
-  url_list = (html ? get_urls_html (input_file, NULL, NULL)
+  url_list = (html ? get_urls_html (input_file, NULL, NULL, iri)
               : get_urls_file (input_file));
 
   for (cur_url = url_list; cur_url; cur_url = cur_url->next, ++*count)
@@ -874,15 +906,16 @@ retrieve_from_file (const char *file, bool html, int *count)
           int old_follow_ftp = opt.follow_ftp;
 
           /* Turn opt.follow_ftp on in case of recursive FTP retrieval */
-          if (cur_url->url->scheme == SCHEME_FTP) 
+          if (cur_url->url->scheme == SCHEME_FTP)
             opt.follow_ftp = 1;
-          
+
           status = retrieve_tree (cur_url->url->url);
 
           opt.follow_ftp = old_follow_ftp;
         }
       else
-        status = retrieve_url (cur_url->url->url, &filename, &new_file, NULL, &dt, opt.recursive);
+        status = retrieve_url (cur_url->url->url, &filename, &new_file, NULL,
+                              &dt, opt.recursive, iri);
 
       if (filename && opt.delete_after && file_exists_p (filename))
         {
@@ -1053,7 +1086,11 @@ bool
 url_uses_proxy (const char *url)
 {
   bool ret;
-  struct url *u = url_parse (url, NULL);
+  struct url *u;
+  struct iri *i = iri_new();
+  /* url was given in the command line, so use locale as encoding */
+  set_uri_encoding (i, opt.locale, true);
+  u= url_parse (url, NULL, i);
   if (!u)
     return false;
   ret = getproxy (u) != NULL;