From: Micah Cowan <micah@cowan.name>
Date: Mon, 15 Oct 2007 02:00:33 +0000 (-0700)
Subject: Josh's patch to remove robots.txt when appriopriate.
X-Git-Tag: v1.13~510
X-Git-Url: http://sjero.net/git/?p=wget;a=commitdiff_plain;h=38b0f4ba27c6c0b2a12d01df50977af05d1a83f4

Josh's patch to remove robots.txt when appriopriate.
---

diff --git a/src/ChangeLog b/src/ChangeLog
index c077ac4e..c4e25c2f 100644
--- a/src/ChangeLog
+++ b/src/ChangeLog
@@ -1,3 +1,8 @@
+2007-10-14  Joshua David Williams  <yurimxpxman@gmail.com>
+
+	* recur.c (download_child_p): Remove robots.txt if
+	--delete-after or --spider is on.
+
 2007-10-13  Micah Cowan  <micah@cowan.name>
 
 	* Makefile.am: Make version.c depend on Wget dependencies (source
diff --git a/src/recur.c b/src/recur.c
index acf0b16d..17f2a2b1 100644
--- a/src/recur.c
+++ b/src/recur.c
@@ -570,6 +570,15 @@ download_child_p (const struct urlpos *upos, struct url *parent, int depth,
           if (res_retrieve_file (url, &rfile))
             {
               specs = res_parse_from_file (rfile);
+
+              /* Delete the robots.txt file if we chose to either delete the
+                 files after downloading or we're just running a spider. */
+              if (opt.delete_after || opt.spider)
+                {
+                  logprintf (LOG_VERBOSE, "Removing %s.\n", rfile);
+                  unlink (rfile);
+                }
+
               xfree (rfile);
             }
           else