[svn] Fix for bug #20296: User:pass@ given in Referer header.

[wget] / src / recur.c

diff --git a/src/recur.c b/src/recur.c
index 611e36061dd0b4eef974a95192c1b011af3a8152..9ecb7d8efb8bfd8bfe33271e0a1d2a8e7c8b283c 100644 (file)
--- a/src/recur.c
+++ b/src/recur.c
@@ -1,11 +1,11 @@
 /* Handling of recursive HTTP retrieving.
-   Copyright (C) 1996-2005 Free Software Foundation, Inc.
+   Copyright (C) 1996-2006 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 
 GNU Wget is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
+the Free Software Foundation; either version 3 of the License, or
  (at your option) any later version.
 
 GNU Wget is distributed in the hope that it will be useful,
@@ -14,8 +14,7 @@ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 GNU General Public License for more details.
 
 You should have received a copy of the GNU General Public License
-along with Wget; if not, write to the Free Software Foundation, Inc.,
-51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+along with Wget.  If not, see <http://www.gnu.org/licenses/>.
 
 In addition, as a special exception, the Free Software Foundation
 gives permission to link the code of its release of Wget with the
@@ -48,6 +47,7 @@ so, delete this exception statement from your version.  */
 #include "hash.h"
 #include "res.h"
 #include "convert.h"
+#include "spider.h"
 \f
 /* Functions for maintaining the URL queue.  */
 
@@ -274,6 +274,11 @@ retrieve_tree (const char *start_url)
            }
        }
 
+      if (opt.spider)
+       {
+          visited_url (url, referer);
+       }
+
       if (descend
          && depth >= opt.reclevel && opt.reclevel != INFINITE_RECURSION)
        {
@@ -319,8 +324,14 @@ retrieve_tree (const char *start_url)
            {
              struct urlpos *child = children;
              struct url *url_parsed = url_parsed = url_parse (url, NULL);
+              char *referer_url = url;
+              bool strip_auth = url_parsed->user;
              assert (url_parsed != NULL);
 
+              /* Strip auth info if present */
+              if (strip_auth)
+                referer_url = url_string (url_parsed, URL_AUTH_HIDE);
+
              for (; child; child = child->next)
                {
                  if (child->ignore_when_downloading)
@@ -331,7 +342,7 @@ retrieve_tree (const char *start_url)
                                        blacklist))
                    {
                      url_enqueue (queue, xstrdup (child->url->url),
-                                  xstrdup (url), depth + 1,
+                                  xstrdup (referer_url), depth + 1,
                                   child->link_expect_html);
                      /* We blacklist the URL we have enqueued, because we
                         don't want to enqueue (and hence download) the
@@ -340,6 +351,8 @@ retrieve_tree (const char *start_url)
                    }
                }
 
+              if (strip_auth)
+                xfree (referer_url);
              url_free (url_parsed);
              free_urlpos (children);
            }
@@ -365,6 +378,7 @@ retrieve_tree (const char *start_url)
                     file);
          if (unlink (file))
            logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
+         logputs (LOG_VERBOSE, "\n");
          register_delete_file (file);
        }
 
@@ -420,6 +434,13 @@ download_child_p (const struct urlpos *upos, struct url *parent, int depth,
 
   if (string_set_contains (blacklist, url))
     {
+      if (opt.spider) 
+       {
+          char *referrer = url_string (parent, URL_AUTH_HIDE_PASSWD);
+          DEBUGP (("download_child_p: parent->url is: `%s'\n", parent->url));
+          visited_url (url, referrer);
+         xfree (referrer);
+       }
       DEBUGP (("Already on the black list.\n"));
       goto out;
     }
@@ -615,3 +636,5 @@ descend_redirect_p (const char *redirected, const char *original, int depth,
 
   return success;
 }
+
+/* vim:set sts=2 sw=2 cino+={s: */