[svn] Merge of fix for bugs 20341 and 20410.

[wget] / src / retr.c

diff --git a/src/retr.c b/src/retr.c
index 1df7f3bfe58a58a094c3ba350f7e1ef736bc9428..c531be59c35bae44344c14e9b661a8c9af05b558 100644 (file)
--- a/src/retr.c
+++ b/src/retr.c
@@ -1,11 +1,11 @@
 /* File retrieval.
 /* File retrieval.

-   Copyright (C) 1996-2005 Free Software Foundation, Inc.
+   Copyright (C) 1996-2006 Free Software Foundation, Inc.

 
 This file is part of GNU Wget.
 
 GNU Wget is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
 
 This file is part of GNU Wget.
 
 GNU Wget is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by

-the Free Software Foundation; either version 2 of the License, or (at
+the Free Software Foundation; either version 3 of the License, or (at

 your option) any later version.
 
 GNU Wget is distributed in the hope that it will be useful,
 your option) any later version.
 
 GNU Wget is distributed in the hope that it will be useful,


@@ -14,8 +14,7 @@ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 GNU General Public License for more details.
 
 You should have received a copy of the GNU General Public License
 GNU General Public License for more details.
 
 You should have received a copy of the GNU General Public License

-along with Wget; if not, write to the Free Software Foundation, Inc.,
-51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+along with Wget.  If not, see <http://www.gnu.org/licenses/>.

 
 In addition, as a special exception, the Free Software Foundation
 gives permission to link the code of its release of Wget with the
 
 In addition, as a special exception, the Free Software Foundation
 gives permission to link the code of its release of Wget with the


@@ -602,7 +601,7 @@ static char *getproxy (struct url *);
 
 uerr_t
 retrieve_url (const char *origurl, char **file, char **newloc,
 
 uerr_t
 retrieve_url (const char *origurl, char **file, char **newloc,

-             const char *refurl, int *dt)
+             const char *refurl, int *dt, bool recursive)

 {
   uerr_t result;
   char *url;
 {
   uerr_t result;
   char *url;


@@ -684,13 +683,12 @@ retrieve_url (const char *origurl, char **file, char **newloc,
       /* If this is a redirection, temporarily turn off opt.ftp_glob
         and opt.recursive, both being undesirable when following
         redirects.  */
       /* If this is a redirection, temporarily turn off opt.ftp_glob
         and opt.recursive, both being undesirable when following
         redirects.  */

-      bool oldrec = opt.recursive, oldglob = opt.ftp_glob;
+      bool oldrec = recursive, glob = opt.ftp_glob;

       if (redirection_count)
       if (redirection_count)

-       opt.recursive = opt.ftp_glob = false;
+       oldrec = glob = false;

 
 

-      result = ftp_loop (u, dt, proxy_url);
-      opt.recursive = oldrec;
-      opt.ftp_glob = oldglob;
+      result = ftp_loop (u, dt, proxy_url, recursive, glob);
+      recursive = oldrec;

 
       /* There is a possibility of having HTTP being redirected to
         FTP.  In these cases we must decide whether the text is HTML
 
       /* There is a possibility of having HTTP being redirected to
         FTP.  In these cases we must decide whether the text is HTML


@@ -845,10 +843,20 @@ retrieve_from_file (const char *file, bool html, int *count)
          break;
        }
       if ((opt.recursive || opt.page_requisites)
          break;
        }
       if ((opt.recursive || opt.page_requisites)

-         && cur_url->url->scheme != SCHEME_FTP)
-       status = retrieve_tree (cur_url->url->url);
+         && (cur_url->url->scheme != SCHEME_FTP || getproxy (cur_url->url)))
+       {
+         int old_follow_ftp = opt.follow_ftp;
+
+         /* Turn opt.follow_ftp on in case of recursive FTP retrieval */
+         if (cur_url->url->scheme == SCHEME_FTP) 
+           opt.follow_ftp = 1;
+         
+         status = retrieve_tree (cur_url->url->url);
+
+         opt.follow_ftp = old_follow_ftp;
+       }

       else
       else

-       status = retrieve_url (cur_url->url->url, &filename, &new_file, NULL, &dt);
+       status = retrieve_url (cur_url->url->url, &filename, &new_file, NULL, &dt, opt.recursive);

 
       if (filename && opt.delete_after && file_exists_p (filename))
        {
 
       if (filename && opt.delete_after && file_exists_p (filename))
        {


@@ -915,9 +923,9 @@ sleep_between_retrievals (int count)
       else
        {
          /* Sleep a random amount of time averaging in opt.wait
       else
        {
          /* Sleep a random amount of time averaging in opt.wait

-            seconds.  The sleeping amount ranges from 0 to
-            opt.wait*2, inclusive.  */
-         double waitsecs = 2 * opt.wait * random_float ();
+            seconds.  The sleeping amount ranges from 0.5*opt.wait to
+            1.5*opt.wait.  */
+         double waitsecs = (0.5 + random_float ()) * opt.wait;

          DEBUGP (("sleep_between_retrievals: avg=%f,sleep=%f\n",
                   opt.wait, waitsecs));
          xsleep (waitsecs);
          DEBUGP (("sleep_between_retrievals: avg=%f,sleep=%f\n",
                   opt.wait, waitsecs));
          xsleep (waitsecs);


@@ -1013,6 +1021,20 @@ getproxy (struct url *u)
   return proxy;
 }
 
   return proxy;
 }
 

+/* Returns true if URL would be downloaded through a proxy. */
+
+bool
+url_uses_proxy (const char *url)
+{
+  bool ret;
+  struct url *u = url_parse (url, NULL);
+  if (!u)
+    return false;
+  ret = getproxy (u) != NULL;
+  url_free (u);
+  return ret;
+}
+

 /* Should a host be accessed through proxy, concerning no_proxy?  */
 static bool
 no_proxy_match (const char *host, const char **no_proxy)
 /* Should a host be accessed through proxy, concerning no_proxy?  */
 static bool
 no_proxy_match (const char *host, const char **no_proxy)