Josh's patch to remove robots.txt when appriopriate.

diff --git a/src/ChangeLog b/src/ChangeLog
index c077ac4e847e6ce0048ac9922545a057ba0c7ee6..c4e25c2f405a99e91580d463285cedba800f0cb8 100644 (file)
--- a/src/ChangeLog
+++ b/src/ChangeLog
@@ -1,3 +1,8 @@
+2007-10-14  Joshua David Williams  <yurimxpxman@gmail.com>
+
+       * recur.c (download_child_p): Remove robots.txt if
+       --delete-after or --spider is on.
+
 2007-10-13  Micah Cowan  <micah@cowan.name>
 
        * Makefile.am: Make version.c depend on Wget dependencies (source

diff --git a/src/recur.c b/src/recur.c
index acf0b16d70db376162ac9a51bb1c49ba6acbe0d6..17f2a2b164d8ae8d48827eca52f60bfdcf40ad59 100644 (file)
--- a/src/recur.c
+++ b/src/recur.c
@@ -570,6 +570,15 @@ download_child_p (const struct urlpos *upos, struct url *parent, int depth,
           if (res_retrieve_file (url, &rfile))
             {
               specs = res_parse_from_file (rfile);
+
+              /* Delete the robots.txt file if we chose to either delete the
+                 files after downloading or we're just running a spider. */
+              if (opt.delete_after || opt.spider)
+                {
+                  logprintf (LOG_VERBOSE, "Removing %s.\n", rfile);
+                  unlink (rfile);
+                }
+
               xfree (rfile);
             }
           else