Check stricter for supported URL scheme.

[wget] / src / retr.c

diff --git a/src/retr.c b/src/retr.c
index 8752dce71510d3ef3819bdbbb4ba7a0906aa1c4a..74b9dccb629a7833908ee4bda7298f93303e43c9 100644 (file)
--- a/src/retr.c
+++ b/src/retr.c
@@ -1,6 +1,6 @@
 /* File retrieval.
-   Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003,
-   2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc.
+   Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
+   2005, 2006, 2007, 2008, 2009 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -39,6 +39,7 @@ as that of the covered work.  */
 #include <string.h>
 #include <assert.h>
 
+#include "exits.h"
 #include "utils.h"
 #include "retr.h"
 #include "progress.h"
@@ -168,7 +169,18 @@ write_data (FILE *out, const char *buf, int bufsize, wgint *skip,
      performance: fast downloads will arrive in large 16K chunks
      (which stdio would write out immediately anyway), and slow
      downloads wouldn't be limited by disk speed.  */
+
+  /* 2005-04-20 SMS.
+     Perhaps it shouldn't hinder performance, but it sure does, at least
+     on VMS (more than 2X).  Rather than speculate on what it should or
+     shouldn't do, it might make more sense to test it.  Even better, it
+     might be nice to explain what possible benefit it could offer, as
+     it appears to be a clear invitation to poor performance with no
+     actual justification.  (Also, why 16K?  Anyone test other values?)
+  */
+#ifndef __VMS
   fflush (out);
+#endif /* ndef __VMS */
   return !ferror (out);
 }
 
@@ -289,7 +301,7 @@ fd_read_body (int fd, FILE *out, wgint toread, wgint startpos,
       else if (ret <= 0)
         break;                  /* EOF or read error */
 
-      if (progress || opt.limit_rate)
+      if (progress || opt.limit_rate || elapsed)
         {
           ptimer_measure (timer);
           if (ret > 0)
@@ -600,11 +612,12 @@ static char *getproxy (struct url *);
 uerr_t
 retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
               char **newloc, const char *refurl, int *dt, bool recursive,
-              struct iri *iri)
+              struct iri *iri, bool register_status)
 {
   uerr_t result;
   char *url;
   bool location_changed;
+  bool iri_fallbacked = 0;
   int dummy;
   char *mynewloc, *proxy;
   struct url *u = orig_parsed, *proxy_url;
@@ -628,15 +641,11 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
   if (file)
     *file = NULL;
 
- second_try:
-  DEBUGP (("[IRI Retrieving %s with %s (UTF-8=%d)\n", quote_n (0, url),
-           iri->uri_encoding ? quote_n (1, iri->uri_encoding) : "None",
-           iri->utf8_encode));
-
   if (!refurl)
     refurl = opt.referer;
 
  redirected:
+  /* (also for IRI fallbacking) */
 
   result = NOCONERROR;
   mynewloc = NULL;
@@ -660,7 +669,8 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
           xfree (url);
           xfree (error);
           RESTORE_POST_DATA;
-          return PROXERR;
+          result = PROXERR;
+          goto bail;
         }
       if (proxy_url->scheme != SCHEME_HTTP && proxy_url->scheme != u->scheme)
         {
@@ -668,7 +678,8 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
           url_free (proxy_url);
           xfree (url);
           RESTORE_POST_DATA;
-          return PROXERR;
+          result = PROXERR;
+          goto bail;
         }
     }
 
@@ -689,7 +700,7 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
       if (redirection_count)
         oldrec = glob = false;
 
-      result = ftp_loop (u, dt, proxy_url, recursive, glob);
+      result = ftp_loop (u, &local_file, dt, proxy_url, recursive, glob);
       recursive = oldrec;
 
       /* There is a possibility of having HTTP being redirected to
@@ -749,7 +760,7 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
           xfree (mynewloc);
           xfree (error);
           RESTORE_POST_DATA;
-          return result;
+          goto bail;
         }
 
       /* Now mynewloc will become newloc_parsed->url, because if the
@@ -771,7 +782,8 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
           xfree (url);
           xfree (mynewloc);
           RESTORE_POST_DATA;
-          return WRONGCODE;
+          result = WRONGCODE;
+          goto bail;
         }
 
       xfree (url);
@@ -805,7 +817,9 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
       if (u)
         {
           DEBUGP (("[IRI fallbacking to non-utf8 for %s\n", quote (url)));
-          goto second_try;
+          url = xstrdup (u->url);
+          iri_fallbacked = 1;
+          goto redirected;
         }
       else
           DEBUGP (("[Couldn't fallback to non-utf8 for %s\n", quote (url)));
@@ -840,7 +854,7 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
       url_free (u);
     }
 
-  if (redirection_count)
+  if (redirection_count || iri_fallbacked)
     {
       if (newloc)
         *newloc = url;
@@ -856,6 +870,9 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
 
   RESTORE_POST_DATA;
 
+bail:
+  if (register_status)
+    inform_exit_status (result);
   return result;
 }
 
@@ -872,7 +889,7 @@ retrieve_from_file (const char *file, bool html, int *count)
   struct urlpos *url_list, *cur_url;
   struct iri *iri = iri_new();
 
-  char *input_file = NULL;
+  char *input_file, *url_file = NULL;
   const char *url = file;
 
   status = RETROK;             /* Suppose everything is OK.  */
@@ -882,7 +899,7 @@ retrieve_from_file (const char *file, bool html, int *count)
   set_uri_encoding (iri, opt.locale, true);
   set_content_encoding (iri, opt.locale);
 
-  if (url_has_scheme (url))
+  if (url_valid_scheme (url))
     {
       int dt,url_err;
       uerr_t status;
@@ -899,9 +916,11 @@ retrieve_from_file (const char *file, bool html, int *count)
       if (!opt.base_href)
         opt.base_href = xstrdup (url);
 
-      status = retrieve_url (url_parsed, url, &input_file, NULL, NULL, &dt,
-                             false, iri);
-      if (status != RETROK)
+      status = retrieve_url (url_parsed, url, &url_file, NULL, NULL, &dt,
+                             false, iri, true);
+      url_free (url_parsed);
+
+      if (!url_file || (status != RETROK))
         return status;
 
       if (dt & TEXTHTML)
@@ -916,6 +935,8 @@ retrieve_from_file (const char *file, bool html, int *count)
       iri->utf8_encode = opt.enable_iri;
       xfree_null (iri->orig_url);
       iri->orig_url = NULL;
+
+      input_file = url_file;
     }
   else
     input_file = (char *) file;
@@ -923,6 +944,8 @@ retrieve_from_file (const char *file, bool html, int *count)
   url_list = (html ? get_urls_html (input_file, NULL, NULL, iri)
               : get_urls_file (input_file));
 
+  xfree_null (url_file);
+
   for (cur_url = url_list; cur_url; cur_url = cur_url->next, ++*count)
     {
       char *filename = NULL, *new_file = NULL;
@@ -960,7 +983,8 @@ retrieve_from_file (const char *file, bool html, int *count)
       else
         status = retrieve_url (parsed_url ? parsed_url : cur_url->url,
                                cur_url->url->url, &filename,
-                               &new_file, NULL, &dt, opt.recursive, tmpiri);
+                               &new_file, NULL, &dt, opt.recursive, tmpiri,
+                               true);
 
       if (parsed_url)
           url_free (parsed_url);