Automated merge.

[wget] / src / retr.c

diff --git a/src/retr.c b/src/retr.c
index 58e00d2fe74102909f78cfdaa57ae5cc7caa304e..7a28ea32e76e825d452417d104fb538a178e79a8 100644 (file)
--- a/src/retr.c
+++ b/src/retr.c
@@ -51,6 +51,7 @@ as that of the covered work.  */
 #include "hash.h"
 #include "convert.h"
 #include "ptimer.h"
+#include "iri.h"
 #include "html-url.h"
 
 /* Total size of downloaded files.  Used to enforce quota.  */
@@ -625,6 +626,9 @@ retrieve_url (const char *origurl, char **file, char **newloc,
   if (file)
     *file = NULL;
 
+  reset_utf8_encode ();
+
+ second_try:
   u = url_parse (url, &up_error_code);
   if (!u)
     {
@@ -633,6 +637,8 @@ retrieve_url (const char *origurl, char **file, char **newloc,
       return URLERROR;
     }
 
+  /*printf ("[Retrieving %s with %s (UTF-8=%d)\n", url, get_remote_charset (), utf8_encoded);*/
+
   if (!refurl)
     refurl = opt.referer;
 
@@ -646,8 +652,11 @@ retrieve_url (const char *origurl, char **file, char **newloc,
   proxy = getproxy (u);
   if (proxy)
     {
+      /* sXXXav : support IRI for proxy */
       /* Parse the proxy URL.  */
+      set_ugly_no_encode (true);
       proxy_url = url_parse (proxy, &up_error_code);
+      set_ugly_no_encode (false);
       if (!proxy_url)
         {
           logprintf (LOG_NOTQUIET, _("Error parsing proxy URL %s: %s.\n"),
@@ -722,6 +731,8 @@ retrieve_url (const char *origurl, char **file, char **newloc,
       xfree (mynewloc);
       mynewloc = construced_newloc;
 
+      reset_utf8_encode ();
+
       /* Now, see if this new location makes sense. */
       newloc_parsed = url_parse (mynewloc, &up_error_code);
       if (!newloc_parsed)
@@ -770,8 +781,21 @@ retrieve_url (const char *origurl, char **file, char **newloc,
       goto redirected;
     }
 
-  if (local_file)
+  /* Try to not encode in UTF-8 if fetching failed */
+  if (!(*dt & RETROKF) && get_utf8_encode ())
     {
+      set_utf8_encode (false);
+      /*printf ("[Fallbacking to non-utf8 for `%s'\n", url);*/
+      goto second_try;
+    }
+
+  if (local_file && *dt & RETROKF)
+    {
+      register_download (u->url, local_file);
+      if (redirection_count && 0 != strcmp (origurl, u->url))
+        register_redirection (origurl, u->url);
+      if (*dt & TEXTHTML)
+        register_html (u->url, local_file);
       if (*dt & RETROKF)
         {
           register_download (u->url, local_file);
@@ -860,9 +884,9 @@ retrieve_from_file (const char *file, bool html, int *count)
           int old_follow_ftp = opt.follow_ftp;
 
           /* Turn opt.follow_ftp on in case of recursive FTP retrieval */
-          if (cur_url->url->scheme == SCHEME_FTP) 
+          if (cur_url->url->scheme == SCHEME_FTP)
             opt.follow_ftp = 1;
-          
+
           status = retrieve_tree (cur_url->url->url);
 
           opt.follow_ftp = old_follow_ftp;
@@ -1039,7 +1063,10 @@ bool
 url_uses_proxy (const char *url)
 {
   bool ret;
-  struct url *u = url_parse (url, NULL);
+  struct url *u;
+  set_ugly_no_encode(true);
+  u= url_parse (url, NULL);
+  set_ugly_no_encode(false);
   if (!u)
     return false;
   ret = getproxy (u) != NULL;