--spider or --delete-after implies --no-directories.

[wget] / src / recur.c

diff --git a/src/recur.c b/src/recur.c
index acf0b16d70db376162ac9a51bb1c49ba6acbe0d6..87440b41c8c7351a89f364673c950d85b3f620d5 100644 (file)
--- a/src/recur.c
+++ b/src/recur.c
@@ -570,6 +570,17 @@ download_child_p (const struct urlpos *upos, struct url *parent, int depth,
           if (res_retrieve_file (url, &rfile))
             {
               specs = res_parse_from_file (rfile);
+
+              /* Delete the robots.txt file if we chose to either delete the
+                 files after downloading or we're just running a spider. */
+              if (opt.delete_after || opt.spider)
+                {
+                  logprintf (LOG_VERBOSE, "Removing %s.\n", rfile);
+                  if (unlink (rfile))
+                      logprintf (LOG_NOTQUIET, "unlink: %s\n",
+                                 strerror (errno));
+                }
+
               xfree (rfile);
             }
           else