Automated merge.

diff --git a/src/ChangeLog b/src/ChangeLog
index 3bae231edd535e5982c64b5080ff77f5a8500781..9cd4d0034e356f57c3097424932ee6aff5fba84f 100644 (file)
--- a/src/ChangeLog
+++ b/src/ChangeLog
@@ -1,3 +1,13 @@
+2007-10-14  Micah Cowan  <micah@cowan.name>
+
+       * recur.c (download_child_p): Print error if unlink of
+       robots.txt fails.
+
+2007-10-14  Joshua David Williams  <yurimxpxman@gmail.com>
+
+       * recur.c (download_child_p): Remove robots.txt if
+       --delete-after or --spider is on.
+
 2007-10-14  Micah Cowan  <micah@cowan.name>
 
        * cmpt.c, cookies.c, ftp-basic.c, ftp-ls.c, ftp.c, hash.c,

diff --git a/src/recur.c b/src/recur.c
index acf0b16d70db376162ac9a51bb1c49ba6acbe0d6..87440b41c8c7351a89f364673c950d85b3f620d5 100644 (file)
--- a/src/recur.c
+++ b/src/recur.c
@@ -570,6 +570,17 @@ download_child_p (const struct urlpos *upos, struct url *parent, int depth,
           if (res_retrieve_file (url, &rfile))
             {
               specs = res_parse_from_file (rfile);
+
+              /* Delete the robots.txt file if we chose to either delete the
+                 files after downloading or we're just running a spider. */
+              if (opt.delete_after || opt.spider)
+                {
+                  logprintf (LOG_VERBOSE, "Removing %s.\n", rfile);
+                  if (unlink (rfile))
+                      logprintf (LOG_NOTQUIET, "unlink: %s\n",
+                                 strerror (errno));
+                }
+
               xfree (rfile);
             }
           else