Josh's patch to remove robots.txt when appriopriate.

[wget] / src / recur.c

diff --git a/src/recur.c b/src/recur.c
index 9de774c44997664ab1da711b61ee79b8aa5517c1..17f2a2b164d8ae8d48827eca52f60bfdcf40ad59 100644 (file)
--- a/src/recur.c
+++ b/src/recur.c
@@ -1,5 +1,6 @@
 /* Handling of recursive HTTP retrieving.
-   Copyright (C) 1996-2006 Free Software Foundation, Inc.
+   Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003,
+   2004, 2005, 2006, 2007 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -325,7 +326,8 @@ retrieve_tree (const char *start_url)
               struct urlpos *child = children;
               struct url *url_parsed = url_parsed = url_parse (url, NULL);
               char *referer_url = url;
-              bool strip_auth = (bool)url_parsed->user;
+              bool strip_auth = (url_parsed != NULL
+                                 && url_parsed->user != NULL);
               assert (url_parsed != NULL);
 
               /* Strip auth info if present */
@@ -568,6 +570,15 @@ download_child_p (const struct urlpos *upos, struct url *parent, int depth,
           if (res_retrieve_file (url, &rfile))
             {
               specs = res_parse_from_file (rfile);
+
+              /* Delete the robots.txt file if we chose to either delete the
+                 files after downloading or we're just running a spider. */
+              if (opt.delete_after || opt.spider)
+                {
+                  logprintf (LOG_VERBOSE, "Removing %s.\n", rfile);
+                  unlink (rfile);
+                }
+
               xfree (rfile);
             }
           else