Eschew config-post.h.

[wget] / src / recur.c

diff --git a/src/recur.c b/src/recur.c
index a8fc71af5fd000ef22acc849dbb620f6f274dde2..8d7b0940488a8f535a3f7a7e96435c26ed9ad814 100644 (file)
--- a/src/recur.c
+++ b/src/recur.c
@@ -1,5 +1,6 @@
 /* Handling of recursive HTTP retrieving.
-   Copyright (C) 1996-2006 Free Software Foundation, Inc.
+   Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003,
+   2004, 2005, 2006, 2007 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -26,7 +27,7 @@ modify this file, you may extend this exception to your version of the
 file, but you are not obligated to do so.  If you do not wish to do
 so, delete this exception statement from your version.  */
 
-#include <config.h>
+#include "wget.h"
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -37,7 +38,6 @@ so, delete this exception statement from your version.  */
 #include <errno.h>
 #include <assert.h>
 
-#include "wget.h"
 #include "url.h"
 #include "recur.h"
 #include "utils.h"
@@ -325,7 +325,8 @@ retrieve_tree (const char *start_url)
               struct urlpos *child = children;
               struct url *url_parsed = url_parsed = url_parse (url, NULL);
               char *referer_url = url;
-              bool strip_auth = url_parsed->user;
+              bool strip_auth = (url_parsed != NULL
+                                 && url_parsed->user != NULL);
               assert (url_parsed != NULL);
 
               /* Strip auth info if present */
@@ -568,6 +569,17 @@ download_child_p (const struct urlpos *upos, struct url *parent, int depth,
           if (res_retrieve_file (url, &rfile))
             {
               specs = res_parse_from_file (rfile);
+
+              /* Delete the robots.txt file if we chose to either delete the
+                 files after downloading or we're just running a spider. */
+              if (opt.delete_after || opt.spider)
+                {
+                  logprintf (LOG_VERBOSE, "Removing %s.\n", rfile);
+                  if (unlink (rfile))
+                      logprintf (LOG_NOTQUIET, "unlink: %s\n",
+                                 strerror (errno));
+                }
+
               xfree (rfile);
             }
           else