X-Git-Url: http://sjero.net/git/?p=wget;a=blobdiff_plain;f=src%2Frecur.c;fp=src%2Frecur.c;h=9fbcafed9cff2ea25578bcc2d2e44f9b1f94cfb9;hp=e32dc64537fcf5db233d5a9e500ccd4c83fe6aa2;hb=462e643a7e31676eceda23e634241f7b4d2cd7bb;hpb=3073a77ed89e7d9527d110a028d134bfd1f9cb40

diff --git a/src/recur.c b/src/recur.c
index e32dc645..9fbcafed 100644
--- a/src/recur.c
+++ b/src/recur.c
@@ -28,7 +28,7 @@ Corresponding Source for a non-source form of such a combination
 shall include the source code for the parts of OpenSSL used as well
 as that of the covered work.  */
 
-#include <config.h>
+#include "wget.h"
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -39,7 +39,6 @@ as that of the covered work.  */
 #include <errno.h>
 #include <assert.h>
 
-#include "wget.h"
 #include "url.h"
 #include "recur.h"
 #include "utils.h"
@@ -571,6 +570,17 @@ download_child_p (const struct urlpos *upos, struct url *parent, int depth,
           if (res_retrieve_file (url, &rfile))
             {
               specs = res_parse_from_file (rfile);
+
+              /* Delete the robots.txt file if we chose to either delete the
+                 files after downloading or we're just running a spider. */
+              if (opt.delete_after || opt.spider)
+                {
+                  logprintf (LOG_VERBOSE, "Removing %s.\n", rfile);
+                  if (unlink (rfile))
+                      logprintf (LOG_NOTQUIET, "unlink: %s\n",
+                                 strerror (errno));
+                }
+
               xfree (rfile);
             }
           else