Josh's patch to remove robots.txt when appriopriate.

[wget] / src / recur.c

diff --git a/src/recur.c b/src/recur.c
index acf0b16d70db376162ac9a51bb1c49ba6acbe0d6..17f2a2b164d8ae8d48827eca52f60bfdcf40ad59 100644 (file)
--- a/src/recur.c
+++ b/src/recur.c
@@ -570,6 +570,15 @@ download_child_p (const struct urlpos *upos, struct url *parent, int depth,
           if (res_retrieve_file (url, &rfile))
             {
               specs = res_parse_from_file (rfile);
+
+              /* Delete the robots.txt file if we chose to either delete the
+                 files after downloading or we're just running a spider. */
+              if (opt.delete_after || opt.spider)
+                {
+                  logprintf (LOG_VERBOSE, "Removing %s.\n", rfile);
+                  unlink (rfile);
+                }
+
               xfree (rfile);
             }
           else