Use Gnulib's alloc functions throughout the source.

[wget] / src / recur.c

diff --git a/src/recur.c b/src/recur.c
index 9de774c44997664ab1da711b61ee79b8aa5517c1..a992230dd1e91fee00229e67b34f613f71fbce17 100644 (file)
--- a/src/recur.c
+++ b/src/recur.c
@@ -1,5 +1,6 @@
 /* Handling of recursive HTTP retrieving.
-   Copyright (C) 1996-2006 Free Software Foundation, Inc.
+   Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003,
+   2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -16,17 +17,20 @@ GNU General Public License for more details.
 You should have received a copy of the GNU General Public License
 along with Wget.  If not, see <http://www.gnu.org/licenses/>.
 
-In addition, as a special exception, the Free Software Foundation
-gives permission to link the code of its release of Wget with the
-OpenSSL project's "OpenSSL" library (or with modified versions of it
-that use the same license as the "OpenSSL" library), and distribute
-the linked executables.  You must obey the GNU General Public License
-in all respects for all of the code used other than "OpenSSL".  If you
-modify this file, you may extend this exception to your version of the
-file, but you are not obligated to do so.  If you do not wish to do
-so, delete this exception statement from your version.  */
+Additional permission under GNU GPL version 3 section 7
 
-#include <config.h>
+If you modify this program, or any covered work, by linking or
+combining it with the OpenSSL project's OpenSSL library (or a
+modified version of that library), containing parts covered by the
+terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
+grants you additional permission to convey the resulting work.
+Corresponding Source for a non-source form of such a combination
+shall include the source code for the parts of OpenSSL used as well
+as that of the covered work.  */
+
+#define USE_GNULIB_ALLOC
+
+#include "wget.h"
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -37,7 +41,6 @@ so, delete this exception statement from your version.  */
 #include <errno.h>
 #include <assert.h>
 
-#include "wget.h"
 #include "url.h"
 #include "recur.h"
 #include "utils.h"
@@ -325,7 +328,8 @@ retrieve_tree (const char *start_url)
               struct urlpos *child = children;
               struct url *url_parsed = url_parsed = url_parse (url, NULL);
               char *referer_url = url;
-              bool strip_auth = (bool)url_parsed->user;
+              bool strip_auth = (url_parsed != NULL
+                                 && url_parsed->user != NULL);
               assert (url_parsed != NULL);
 
               /* Strip auth info if present */
@@ -437,7 +441,7 @@ download_child_p (const struct urlpos *upos, struct url *parent, int depth,
       if (opt.spider) 
         {
           char *referrer = url_string (parent, URL_AUTH_HIDE_PASSWD);
-          DEBUGP (("download_child_p: parent->url is: `%s'\n", parent->url));
+          DEBUGP (("download_child_p: parent->url is: %s\n", quote (parent->url)));
           visited_url (url, referrer);
           xfree (referrer);
         }
@@ -568,6 +572,17 @@ download_child_p (const struct urlpos *upos, struct url *parent, int depth,
           if (res_retrieve_file (url, &rfile))
             {
               specs = res_parse_from_file (rfile);
+
+              /* Delete the robots.txt file if we chose to either delete the
+                 files after downloading or we're just running a spider. */
+              if (opt.delete_after || opt.spider)
+                {
+                  logprintf (LOG_VERBOSE, "Removing %s.\n", rfile);
+                  if (unlink (rfile))
+                      logprintf (LOG_NOTQUIET, "unlink: %s\n",
+                                 strerror (errno));
+                }
+
               xfree (rfile);
             }
           else